From 58d868c8cb41cf5d2f3ff3ea38906721430b5b74 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 08:51:19 +0200 Subject: [PATCH 001/221] added stats extractor parent component added PlainExtractor based on numpy and scipy functions --- src/ctapipe/calib/camera/extractor.py | 86 +++++++++++++++++++++++++++ src/ctapipe/containers.py | 3 + 2 files changed, 89 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..0140ed5bcb4 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,86 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", +] + + +from abc import abstractmethod + +import numpy as np +import scipy.stats +from traitlets import Int + +from ctapipe.core import TelescopeComponent +from ctapipe.containers import StatisticsContainer + + +class StatisticsExtractor(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__(self, images, trigger_times) -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + images : ndarray + images stored in a numpy array of shape + (n_images, n_channels, n_pix). + trigger_times : ndarray + images stored in a numpy array of shape + (n_images, ) + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(img, time)) + + return stats_list + + def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=np.nanmean(images, axis=0), + median=np.nanmedian(images, axis=0), + std=np.nanstd(images, axis=0), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 9a5b39f4da8..1f2334ba386 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -414,9 +414,12 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics""" + validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") From edfccc638ac442550eabbc410fafc10fef4ecad2 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 15:51:47 +0200 Subject: [PATCH 002/221] added stats extractor based on sigma clipping --- src/ctapipe/calib/camera/extractor.py | 67 ++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0140ed5bcb4..654be103f8b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,6 +5,7 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "SigmaClippingExtractor", ] @@ -12,10 +13,14 @@ import numpy as np import scipy.stats -from traitlets import Int +from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core.traits import ( + Int, + List, +) class StatisticsExtractor(TelescopeComponent): @@ -84,3 +89,63 @@ def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: skewness=scipy.stats.skew(images, axis=0), kurtosis=scipy.stats.kurtosis(images, axis=0), ) + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(img, time)) + + return stats_list + + def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + + # mean, median, and std over the sample per pixel + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + std=pixel_std.filled(np.nan), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) + From d19168c47dafd5ff96d3c866ee10cf1a03d6cbef Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Tue, 30 Apr 2024 16:34:57 +0200 Subject: [PATCH 003/221] added cut of outliers restructured the stats containers --- src/ctapipe/calib/camera/extractor.py | 139 +++++++++++++++------ src/ctapipe/containers.py | 17 ++- src/ctapipe/image/statistics.py | 6 +- src/ctapipe/image/tests/test_statistics.py | 4 +- 4 files changed, 119 insertions(+), 47 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 654be103f8b..6e7ca6aa634 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -24,6 +24,17 @@ class StatisticsExtractor(TelescopeComponent): + + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics @@ -36,19 +47,18 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, images, trigger_times) -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: """ Call the relevant functions to extract the statistics for the particular extractor. Parameters ---------- - images : ndarray - images stored in a numpy array of shape + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). - trigger_times : ndarray - images stored in a numpy array of shape - (n_images, ) + col_name : string + column name in the dl1 table Returns ------- @@ -62,42 +72,60 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - sample_size = Int(2500, help="sample size").tag(config=True) + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: - def __call__(self, dl1_table, col_name="image") -> list: - # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), - mean=np.nanmean(images, axis=0), - median=np.nanmedian(images, axis=0), - std=np.nanstd(images, axis=0), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using astropy's sigma clipping functions """ - sample_size = Int(2500, help="sample size").tag(config=True) - sigma_clipping_max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", @@ -107,8 +135,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - - def __call__(self, dl1_table, col_name="image") -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) @@ -116,17 +143,26 @@ def __call__(self, dl1_table, col_name="image") -> list: # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, + masked_images, + sigma=max_sigma, maxiters=self.sigma_clipping_iterations, cenfunc="mean", axis=0, @@ -137,15 +173,42 @@ def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContaine pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), std=pixel_std.filled(np.nan), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + std_outliers=image_std_outliers.filled(True), ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 1f2334ba386..b8ce24e3973 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", "SchedulingBlockContainer", @@ -412,24 +413,32 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics""" + """Store descriptive statistics of a sequence of images""" validity_start = Field(np.float32(nan), "start") validity_stop = Field(np.float32(nan), "stop") + mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") + median_outliers = Field(np.float32(nan), "median intensity") + std = Field(np.float32(nan), "standard deviation of intensity") + std_outliers = Field(np.float32(nan), "standard deviation intensity") + +class ImageStatisticsContainer(Container): + """Store descriptive image statistics""" + max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") -class IntensityStatisticsContainer(StatisticsContainer): +class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" -class PeakTimeStatisticsContainer(StatisticsContainer): +class PeakTimeStatisticsContainer(ImageStatisticsContainer): default_prefix = "peak_time" diff --git a/src/ctapipe/image/statistics.py b/src/ctapipe/image/statistics.py index ee8ddbd8aed..966b3bbfa76 100644 --- a/src/ctapipe/image/statistics.py +++ b/src/ctapipe/image/statistics.py @@ -1,7 +1,7 @@ import numpy as np from numba import njit -from ..containers import StatisticsContainer +from ..containers import ImageStatisticsContainer __all__ = ["descriptive_statistics", "skewness", "kurtosis"] @@ -79,8 +79,8 @@ def kurtosis(data, mean=None, std=None, fisher=True): def descriptive_statistics( - values, container_class=StatisticsContainer -) -> StatisticsContainer: + values, container_class=ImageStatisticsContainer +) -> ImageStatisticsContainer: """compute intensity statistics of an image""" mean = values.mean() std = values.std() diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index b9773edc9a7..ca8d9d4dbf4 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,14 +49,14 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, StatisticsContainer + from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) data = rng.normal(5, 2, 1000) stats = descriptive_statistics(data) - assert isinstance(stats, StatisticsContainer) + assert isinstance(stats, ImageStatisticsContainer) stats = descriptive_statistics(data, container_class=PeakTimeStatisticsContainer) assert isinstance(stats, PeakTimeStatisticsContainer) From 49f0f8a21c6fc085fb871292a0e655ff62c9a743 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:10:59 +0200 Subject: [PATCH 004/221] update docs --- src/ctapipe/calib/camera/extractor.py | 4 +++- src/ctapipe/containers.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 6e7ca6aa634..56b8a7f2219 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,7 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) + sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -57,6 +57,8 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> dl1_table : ndarray dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). + masked_pixels_of_sample : ndarray + boolean array of masked pixels that are not available for processing col_name : string column name in the dl1 table diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index b8ce24e3973..5be78775580 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -415,13 +415,13 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") - mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") - median_outliers = Field(np.float32(nan), "median intensity") - std = Field(np.float32(nan), "standard deviation of intensity") - std_outliers = Field(np.float32(nan), "standard deviation intensity") + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + median = Field(np.float32(nan), "Channel-wise and pixel-wise median") + median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") + std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") + std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 9035fb43e9d312e5c208717fb6f2e596e4e5d5e5 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:12:33 +0200 Subject: [PATCH 005/221] formatted with black --- src/ctapipe/calib/camera/extractor.py | 87 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 56b8a7f2219..907f22923d2 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,10 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) + sample_size = Int( + 2500, + help="Size of the sample used for the calculation of the statistical values", + ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -47,7 +50,9 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -68,31 +73,41 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> List of extracted statistics and validity ranges """ + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using numpy and scipy functions """ - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -137,25 +152,34 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median of the event images image_median = np.ma.median(masked_images, axis=-1) @@ -194,15 +218,21 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = ( - np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], - image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = ( - np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) return StatisticsContainer( validity_start=times[0], @@ -213,4 +243,3 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S std=pixel_std.filled(np.nan), std_outliers=image_std_outliers.filled(True), ) - From 8a0e89f5b54da2df3d1c383937f8eab22ff69637 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:29:10 +0200 Subject: [PATCH 006/221] added pass for __call__ function --- src/ctapipe/calib/camera/extractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 907f22923d2..e4ed0342e1b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -72,6 +72,7 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ + pass class PlainExtractor(StatisticsExtractor): From 82b29e9f8397e8a6ca1e9e2cd1e753d34aa602ff Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:41:44 +0200 Subject: [PATCH 007/221] added unit tests --- .../calib/camera/tests/test_extractors.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..5e5f7a617fc --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,51 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import astropy.units as u +from astropy.table import QTable +import numpy as np + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +def test_extractors(example_subarray): + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False + +def test_check_outliers(example_subarray): + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:,0,120] = 120.0 + flatfield_dl1_data[:,1,67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + #check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] == True + assert sigmaclipping_stats_list[0].median_outliers[1][67] == True + assert sigmaclipping_stats_list[1].median_outliers[0][120] == True + assert sigmaclipping_stats_list[1].median_outliers[1][67] == True From 1805cb25eaf5abb63fc1aa3e880ffef29e9dd917 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:50:48 +0200 Subject: [PATCH 008/221] added changelog --- docs/changes/2554.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/2554.feature.rst diff --git a/docs/changes/2554.feature.rst b/docs/changes/2554.feature.rst new file mode 100644 index 00000000000..2e6a6356b3a --- /dev/null +++ b/docs/changes/2554.feature.rst @@ -0,0 +1 @@ +Add API to extract the statistics from a sequence of images. From a4cea6a7ee6d42171e6addd4a59df19066f5fba6 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 13:54:45 +0200 Subject: [PATCH 009/221] fix lint --- src/ctapipe/calib/camera/extractor.py | 9 --------- src/ctapipe/calib/camera/tests/test_extractors.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index e4ed0342e1b..7b37fb53b1c 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -119,9 +119,6 @@ def _plain_extraction( # std over the sample per pixel pixel_std = np.ma.std(masked_images, axis=0) - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - # outliers from median image_median_outliers = np.logical_or( pixel_median < self.image_median_cut_outliers[0], @@ -182,9 +179,6 @@ def _sigmaclipping_extraction( # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - # median of the event images - image_median = np.ma.median(masked_images, axis=-1) - # mean, median, and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( @@ -202,9 +196,6 @@ def _sigmaclipping_extraction( unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - # only warn for values discard in the sigma clipping, not those from before - outliers = unused_values & (~masked_images.mask) - # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 5e5f7a617fc..19b04c017fe 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,7 +22,7 @@ def test_extractors(example_subarray): plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False From 16e5920ba9a8e83e2c06eb77e163262f112e9d6a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 7 May 2024 09:30:29 +0200 Subject: [PATCH 010/221] Small commit for prototyping --- src/ctapipe/calib/camera/extractor.py | 46 ++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index e4ed0342e1b..718aa48494b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -6,6 +6,7 @@ "StatisticsExtractor", "PlainExtractor", "SigmaClippingExtractor", + "StarExtractor", ] @@ -14,6 +15,8 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats +from astropy.coordinates import EarthLocation, SkyCoord, Angle +from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -21,6 +24,7 @@ Int, List, ) +from ctapipe.coordinates import EngineeringCameraFrame class StatisticsExtractor(TelescopeComponent): @@ -138,9 +142,49 @@ def _plain_extraction( ) +class StarExtractor(StatisticsExtractor): + """ + Extracts pointing information from a series of variance images + using the startracker functions + """ + + min_star_magnitude = Float( + 0.1, + help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + ).tag(config=True) + + def __init__(): + + def __call__( + self, variance_table, initial_pointing, PSF_model + ): + + def _stars_in_FOV( + self, pointing + ): + + stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] + + for star in stars: + + star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') + star_coords = star_coords.transform_to(camera_frame) + central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( + + def _star_extraction( + self, + ): + camera_frame = EngineeringCameraFrame( + telescope_pointing=current_pointing, + focal_length=self.focal_length, + obstime=time.utc, + + + + class SigmaClippingExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extracts the statistics from a sequence of images using astropy's sigma clipping functions """ From 63d177431381d44db08b02524ef32e60452ccc17 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 10 May 2024 09:05:01 +0200 Subject: [PATCH 011/221] Removed unneeded functions --- src/ctapipe/calib/camera/extractor.py | 28 +++------------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 718aa48494b..eb245447527 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,8 +5,8 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "StarVarianceExtractor", "SigmaClippingExtractor", - "StarExtractor", ] @@ -15,8 +15,6 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats -from astropy.coordinates import EarthLocation, SkyCoord, Angle -from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -142,7 +140,7 @@ def _plain_extraction( ) -class StarExtractor(StatisticsExtractor): +class StarVarianceExtractor(StatisticsExtractor): """ Extracts pointing information from a series of variance images using the startracker functions @@ -156,29 +154,9 @@ class StarExtractor(StatisticsExtractor): def __init__(): def __call__( - self, variance_table, initial_pointing, PSF_model + self, variance_table, trigger_table, initial_pointing, PSF_model ): - def _stars_in_FOV( - self, pointing - ): - - stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] - - for star in stars: - - star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') - star_coords = star_coords.transform_to(camera_frame) - central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( - - def _star_extraction( - self, - ): - camera_frame = EngineeringCameraFrame( - telescope_pointing=current_pointing, - focal_length=self.focal_length, - obstime=time.utc, - From e6ba1776ee9309de1153270e842eb1bb19d47c3a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 23 May 2024 11:37:46 +0200 Subject: [PATCH 012/221] I altered the class variables to th evariance statistics extractor --- src/ctapipe/calib/camera/extractor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index c8a1b3158ce..70f4a2eb561 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -143,20 +143,23 @@ class StarVarianceExtractor(StatisticsExtractor): using the startracker functions """ - min_star_magnitude = Float( - 0.1, - help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) def __init__(): def __call__( - self, variance_table, trigger_table, initial_pointing, PSF_model + self, variance_table ): - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images From ddf342b973b1785b1a37f78c5ff021d06f87bad7 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 24 May 2024 13:38:36 +0200 Subject: [PATCH 013/221] added a container for mean variance images and fixed docustring --- src/ctapipe/calib/camera/extractor.py | 43 +++++++++++++++++++++++++-- src/ctapipe/containers.py | 9 +++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 70f4a2eb561..3cc51de40cb 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -139,8 +139,9 @@ def _plain_extraction( class StarVarianceExtractor(StatisticsExtractor): """ - Extracts pointing information from a series of variance images - using the startracker functions + Generating average variance images from a set + of variance images for the startracker + pointing calibration """ sigma_clipping_max_sigma = Int( @@ -158,7 +159,43 @@ def __call__( self, variance_table ): - + image_chunks = ( + variance_table["image"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["image"].data), self.sample_size) + ) + + time_chunks = ( + variance_table["trigger_times"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) + ) + + stats_list = [] + + for images, times in zip(image_chunks, time_chunks): + + stats_list.append( + self._sigmaclipping_extraction(images, times) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times + )->StatisticsContainer: + + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan) + ) class SigmaClippingExtractor(StatisticsExtractor): """ diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 5be78775580..0aeff5a97eb 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -423,6 +424,13 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") +class VarianceStatisticsContainer(Container): + """Store descriptive statistics of a sequence of variance images""" + + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + class ImageStatisticsContainer(Container): """Store descriptive image statistics""" @@ -433,7 +441,6 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") - class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From 0beb975a184b2f5b3502d5cf0a263aa1c0ed512e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 27 May 2024 15:20:53 +0200 Subject: [PATCH 014/221] I changed the container type for the StarVarianceExtractor --- src/ctapipe/calib/camera/extractor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 3cc51de40cb..ffcd561aa87 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -180,7 +180,7 @@ def __call__( def _sigmaclipping_extraction( self, images, times - )->StatisticsContainer: + )->VarianceStatisticsContainer: pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( images, @@ -190,11 +190,10 @@ def _sigmaclipping_extraction( axis=0, ) - return StatisticsContainer( + return VarianceStatisticsContainer( validity_start=times[0], validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan) + mean=pixel_mean.filled(np.nan) ) class SigmaClippingExtractor(StatisticsExtractor): From b4df919f9040ad100635fe4b3b2f5ea865305019 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:42:35 +0200 Subject: [PATCH 015/221] fix pylint Remove StarVarianceExtractor since is functionality is featured in the existing Extractors --- src/ctapipe/calib/camera/extractor.py | 89 ++++--------------- .../calib/camera/tests/test_extractors.py | 64 +++++++------ src/ctapipe/containers.py | 7 -- 3 files changed, 53 insertions(+), 107 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index ffcd561aa87..9e9e9947462 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,15 +5,12 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", - "StarVarianceExtractor", "SigmaClippingExtractor", ] - from abc import abstractmethod import numpy as np -import scipy.stats from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent @@ -22,10 +19,9 @@ Int, List, ) -from ctapipe.coordinates import EngineeringCameraFrame - class StatisticsExtractor(TelescopeComponent): + """Base StatisticsExtractor component""" sample_size = Int( 2500, @@ -33,11 +29,13 @@ class StatisticsExtractor(TelescopeComponent): ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", + help="""Interval of accepted image values \\ + (fraction with respect to camera median value)""", ).tag(config=True) image_std_cut_outliers = List( [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", + help="""Interval (number of std) of accepted image standard deviation \\ + around camera median value""", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): @@ -74,7 +72,6 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ - pass class PlainExtractor(StatisticsExtractor): @@ -136,66 +133,6 @@ def _plain_extraction( std=pixel_std.filled(np.nan), ) - -class StarVarianceExtractor(StatisticsExtractor): - """ - Generating average variance images from a set - of variance images for the startracker - pointing calibration - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __init__(): - - def __call__( - self, variance_table - ): - - image_chunks = ( - variance_table["image"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["image"].data), self.sample_size) - ) - - time_chunks = ( - variance_table["trigger_times"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) - ) - - stats_list = [] - - for images, times in zip(image_chunks, time_chunks): - - stats_list.append( - self._sigmaclipping_extraction(images, times) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times - )->VarianceStatisticsContainer: - - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - return VarianceStatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan) - ) - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images @@ -273,18 +210,26 @@ def _sigmaclipping_extraction( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], ) return StatisticsContainer( diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 19b04c017fe..89c375387e3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,7 +2,6 @@ Tests for StatisticsExtractor and related functions """ -import astropy.units as u from astropy.table import QTable import numpy as np @@ -10,42 +9,51 @@ def test_extractors(example_subarray): + """test basic functionality of the StatisticsExtractors""" + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) - + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False - + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + + def test_check_outliers(example_subarray): - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + """test detection ability of outliers""" + + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers - flatfield_dl1_data[:,0,120] = 120.0 - flatfield_dl1_data[:,1,67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - #check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] == True - assert sigmaclipping_stats_list[0].median_outliers[1][67] == True - assert sigmaclipping_stats_list[1].median_outliers[0][120] == True - assert sigmaclipping_stats_list[1].median_outliers[1][67] == True + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] is True + assert sigmaclipping_stats_list[0].median_outliers[1][67] is True + assert sigmaclipping_stats_list[1].median_outliers[0][120] is True + assert sigmaclipping_stats_list[1].median_outliers[1][67] is True diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 0aeff5a97eb..68685a26623 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,7 +57,6 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", - "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -424,12 +423,6 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") -class VarianceStatisticsContainer(Container): - """Store descriptive statistics of a sequence of variance images""" - - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 3483fd4262ea2dd40e4970e7955b0139dc2975a7 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:45:29 +0200 Subject: [PATCH 016/221] change __call__() to _extract() --- src/ctapipe/calib/camera/extractor.py | 6 +++--- src/ctapipe/calib/camera/tests/test_extractors.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 9e9e9947462..eaff6c714c5 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -80,7 +80,7 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: @@ -148,7 +148,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 89c375387e3..d5f082762ed 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,8 +22,8 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -50,7 +50,7 @@ def test_check_outliers(example_subarray): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From 5bbedda2ec700236884807a6c2261537ee02db95 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:59:36 +0200 Subject: [PATCH 017/221] minor renaming --- src/ctapipe/calib/camera/extractor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index eaff6c714c5..0c23caebb00 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -60,7 +60,7 @@ def _extract( Parameters ---------- dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape + dl1 table with images and timestamps stored in a numpy array of shape (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing @@ -139,11 +139,11 @@ class SigmaClippingExtractor(StatisticsExtractor): using astropy's sigma clipping functions """ - sigma_clipping_max_sigma = Int( + max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", ).tag(config=True) - sigma_clipping_iterations = Int( + iterations = Int( default_value=5, help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) @@ -178,11 +178,10 @@ def _sigmaclipping_extraction( masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, + sigma=self.max_sigma, + maxiters=self.iterations, cenfunc="mean", axis=0, ) @@ -192,7 +191,7 @@ def _sigmaclipping_extraction( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 14a0910519bb2b12cd59f31f2cfbadc30681497c Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:15:53 +0200 Subject: [PATCH 018/221] use pytest.fixture for Extractors --- .../calib/camera/tests/test_extractors.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d5f082762ed..d363ee24ff0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -7,14 +7,21 @@ from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor +@pytest.fixture + def test_plainextractor(example_subarray): + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -def test_extractors(example_subarray): +@pytest.fixture + def test_sigmaclippingextractor(example_subarray): + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" - plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) @@ -22,8 +29,10 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + dl1_table=flatfield_dl1_table + ) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -38,19 +47,18 @@ def test_extractors(example_subarray): assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False -def test_check_outliers(example_subarray): +def test_check_outliers(test_sigmaclippingextractor): """test detection ability of outliers""" - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract( + dl1_table=flatfield_dl1_table + ) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From f126748e9817f023d7c9173e35779e53517aab6c Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:59:57 +0200 Subject: [PATCH 019/221] reduce duplicated code of the call function --- src/ctapipe/calib/camera/extractor.py | 52 ++++++------------- .../calib/camera/tests/test_extractors.py | 30 ++++++----- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0c23caebb00..d060d620508 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -49,8 +49,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - @abstractmethod - def _extract( + def __call__( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -73,17 +72,6 @@ def _extract( List of extracted statistics and validity ranges """ - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] @@ -98,11 +86,23 @@ def _extract( stats_list = [] for images, times in zip(image_chunks, time_chunks): stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) + self._extract(images, times, masked_pixels_of_sample) ) return stats_list - def _plain_extraction( + @abstractmethod + def _extract( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + pass + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def _extract( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: @@ -149,28 +149,6 @@ class SigmaClippingExtractor(StatisticsExtractor): ).tag(config=True) def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d363ee24ff0..06107a6e7b7 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -4,20 +4,22 @@ from astropy.table import QTable import numpy as np - +import pytest from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor -@pytest.fixture - def test_plainextractor(example_subarray): - return PlainExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -@pytest.fixture - def test_sigmaclippingextractor(example_subarray): - return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -29,8 +31,8 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) @@ -56,7 +58,7 @@ def test_check_outliers(test_sigmaclippingextractor): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract( + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) From 0ec81b9d3469bd6a688e9f73a09789f08e588f36 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 12:02:41 +0200 Subject: [PATCH 020/221] edit description of StatisticsContainer --- src/ctapipe/containers.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 68685a26623..f61e429417d 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -415,13 +415,33 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") - median = Field(np.float32(nan), "Channel-wise and pixel-wise median") - median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") - std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") - std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") + extraction_start = Field(np.float32(nan), "start of the extraction sequence") + extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + mean = Field( + None, + "mean of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median = Field( + None, + "median of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median_outliers = Field( + None, + "outliers from the median distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) + std = Field( + None, + "standard deviation of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + std_outliers = Field( + None, + "outliers from the standard deviation distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) class ImageStatisticsContainer(Container): @@ -434,6 +454,7 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") + class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From 3910c22c969840005d8dfa9d07e469fb2432ec4a Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 13:12:54 +0200 Subject: [PATCH 021/221] added feature to shift the extraction sequence allow overlapping extraction sequences --- src/ctapipe/calib/camera/extractor.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index d060d620508..f4ddb03d4bd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,11 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" + self, + dl1_table, + masked_pixels_of_sample=None, + sample_shift=None, + col_name="image", ) -> list: """ Call the relevant functions to extract the statistics @@ -63,6 +67,8 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing + sample_shift : int + number of samples to shift the extraction sequence col_name : string column name in the dl1 table @@ -72,14 +78,19 @@ def __call__( List of extracted statistics and validity ranges """ + # If no sample_shift is provided, the sample_shift is set to self.sample_size + # meaning that the samples are not overlapping. + if sample_shift is None: + sample_shift = self.sample_size + # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) + for i in range(0, len(dl1_table[col_name].data), sample_shift) ) time_chunks = ( dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) + for i in range(0, len(dl1_table["time"]), sample_shift) ) # Calculate the statistics from a sequence of images @@ -169,7 +180,9 @@ def _extract( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > ( + self.max_sigma * pixel_std + ) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 5c6595b2dff71d93c3fbcfe354aa31017997229d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 14:23:53 +0200 Subject: [PATCH 022/221] fix boundary case for the last chunk renaming to chunk(s) and chunk_size and _shift added test for chunk_shift and boundary case --- src/ctapipe/calib/camera/extractor.py | 72 ++++++++++--------- .../calib/camera/tests/test_extractors.py | 21 +++++- src/ctapipe/containers.py | 6 +- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index f4ddb03d4bd..86d9c1345fd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,5 +1,5 @@ """ -Extraction algorithms to compute the statistics from a sequence of images +Extraction algorithms to compute the statistics from a chunk of images """ __all__ = [ @@ -23,9 +23,9 @@ class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" - sample_size = Int( + chunk_size = Int( 2500, - help="Size of the sample used for the calculation of the statistical values", + help="Size of the chunk used for the calculation of the statistical values", ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], @@ -41,7 +41,7 @@ class StatisticsExtractor(TelescopeComponent): def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). + from a chunk of charges and pulse times (images). Parameters ---------- @@ -53,7 +53,7 @@ def __call__( self, dl1_table, masked_pixels_of_sample=None, - sample_shift=None, + chunk_shift=None, col_name="image", ) -> list: """ @@ -67,38 +67,44 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing - sample_shift : int - number of samples to shift the extraction sequence + chunk_shift : int + number of samples to shift the extraction chunk col_name : string column name in the dl1 table Returns ------- List StatisticsContainer: - List of extracted statistics and validity ranges + List of extracted statistics and extraction chunks """ - # If no sample_shift is provided, the sample_shift is set to self.sample_size - # meaning that the samples are not overlapping. - if sample_shift is None: - sample_shift = self.sample_size - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), sample_shift) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), sample_shift) - ) - - # Calculate the statistics from a sequence of images + # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size + # meaning that the extraction chunks are not overlapping. + if chunk_shift is None: + chunk_shift = self.chunk_size + + # Function to split table data into appropriated chunks + def _get_chunks(col_name): + return [ + ( + dl1_table[col_name].data[i : i + self.chunk_size] + if i + self.chunk_size <= len(dl1_table[col_name]) + else dl1_table[col_name].data[ + len(dl1_table[col_name].data) + - self.chunk_size : len(dl1_table[col_name].data) + ] + ) + for i in range(0, len(dl1_table[col_name].data), chunk_shift) + ] + + # Get the chunks for the timestamps and selected column name + time_chunks = _get_chunks("time") + image_chunks = _get_chunks(col_name) + + # Calculate the statistics from a chunk of images stats_list = [] for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._extract(images, times, masked_pixels_of_sample) - ) + stats_list.append(self._extract(images, times, masked_pixels_of_sample)) return stats_list @abstractmethod @@ -109,7 +115,7 @@ def _extract( class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extractor the statistics from a chunk of images using numpy and scipy functions """ @@ -136,8 +142,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), @@ -146,7 +152,7 @@ def _extract( class SigmaClippingExtractor(StatisticsExtractor): """ - Extracts the statistics from a sequence of images + Extracts the statistics from a chunk of images using astropy's sigma clipping functions """ @@ -223,8 +229,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 06107a6e7b7..40efd4f2fc3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -11,14 +11,14 @@ def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" return PlainExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): @@ -67,3 +67,20 @@ def test_check_outliers(test_sigmaclippingextractor): assert sigmaclipping_stats_list[0].median_outliers[1][67] is True assert sigmaclipping_stats_list[1].median_outliers[0][120] is True assert sigmaclipping_stats_list[1].median_outliers[1][67] is True + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, + chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 + diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index f61e429417d..dcd70255519 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -413,10 +413,10 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a sequence of images""" + """Store descriptive statistics of a chunk of images""" - extraction_start = Field(np.float32(nan), "start of the extraction sequence") - extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + extraction_start = Field(np.float32(nan), "start of the extraction chunk") + extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") mean = Field( None, "mean of a pixel-wise quantity for each channel" From bc65984855b5f4bb10c263d6deb7612099580ed0 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 023/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 223 ++++++++++++++++++++++++- src/ctapipe/image/psf_model.py | 80 +++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index baf3d2f1057..81cacedc432 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,24 +2,33 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Integer, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource -__all__ = ["CameraCalibrator"] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -47,6 +56,218 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) + + @abstractmethod + def __call__(self, data_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated + tel_id : int + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties + """ + + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) + + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): + """ + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + pass + + +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + PSFModel_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="PlainExtractor", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + self.psf = PSFModel.from_name( + self.PSFModel_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, url, tel_id): + if self._check_req_data(url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + with EventSource(url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + def _calibrate_varimages(self, varimages, gain): + pass + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..7af526e6c21 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,80 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + +from ctapipe.core import TelescopeComponent + + +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base Component to describe image distortion due to the optics of the different cameras. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) From d7a65a34d5b92309f6cb98ea9ab41e14725f6ecc Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 024/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 81cacedc432..cd7ad324e1f 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -266,6 +266,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From ba00bf3c4c080de3a0c674b99732e7dc6e6d7de6 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:13:27 +0200 Subject: [PATCH 025/221] fix tests --- .../calib/camera/tests/test_extractors.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 40efd4f2fc3..a83c93fd1c0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,24 +2,24 @@ Tests for StatisticsExtractor and related functions """ -from astropy.table import QTable import numpy as np import pytest +from astropy.table import QTable + from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + @pytest.fixture(name="test_plainextractor") def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" - return PlainExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" - return SigmaClippingExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -36,17 +36,17 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): dl1_table=flatfield_dl1_table ) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) def test_check_outliers(test_sigmaclippingextractor): @@ -63,11 +63,11 @@ def test_check_outliers(test_sigmaclippingextractor): ) # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] is True - assert sigmaclipping_stats_list[0].median_outliers[1][67] is True - assert sigmaclipping_stats_list[1].median_outliers[0][120] is True - assert sigmaclipping_stats_list[1].median_outliers[1][67] is True - + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + def test_check_chunk_shift(test_sigmaclippingextractor): """test the chunk shift option and the boundary case for the last chunk""" @@ -77,10 +77,8 @@ def test_check_chunk_shift(test_sigmaclippingextractor): # insert outliers flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, - chunk_shift=2000 + dl1_table=flatfield_dl1_table, chunk_shift=2000 ) # check if three chunks are used for the extraction assert len(sigmaclipping_stats_list) == 3 - From d3aae3289a2deef1fe5a815d2ea547c04690859e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:26:03 +0200 Subject: [PATCH 026/221] fix ruff --- src/ctapipe/calib/camera/extractor.py | 40 +++++++++------------- src/ctapipe/image/tests/test_statistics.py | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 86d9c1345fd..4c8f49d1f38 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -13,13 +13,14 @@ import numpy as np from astropy.stats import sigma_clipped_stats -from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Int, List, ) + class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" @@ -90,8 +91,9 @@ def _get_chunks(col_name): dl1_table[col_name].data[i : i + self.chunk_size] if i + self.chunk_size <= len(dl1_table[col_name]) else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - - self.chunk_size : len(dl1_table[col_name].data) + len(dl1_table[col_name].data) - self.chunk_size : len( + dl1_table[col_name].data + ) ] ) for i in range(0, len(dl1_table[col_name].data), chunk_shift) @@ -108,21 +110,17 @@ def _get_chunks(col_name): return stats_list @abstractmethod - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: pass + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a chunk of images using numpy and scipy functions """ - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -150,6 +148,7 @@ def _extract( std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a chunk of images @@ -165,10 +164,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -206,25 +202,21 @@ def _extract( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], ) diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index ca8d9d4dbf4..2ebe149b57b 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer + from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From 825b25b32f98bd7f3f4b81be1248ea7b01d6465a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 027/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index cd7ad324e1f..0c48113e4ae 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -111,21 +111,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -208,7 +220,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -224,7 +236,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -264,7 +276,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From a5b878dd63a7438e60adbc5c953de546bfd82e2c Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 028/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 0c48113e4ae..f921b2d97fb 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -276,9 +276,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From 78481ccf0c9e08ae33d36a96fa41f5b6ba799fbe Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 029/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index f921b2d97fb..360103982a8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -26,7 +26,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -270,6 +270,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -294,7 +299,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From 41a42466331c1086a27a9682be89661ada061750 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 030/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 227 +++++++++++++++++-------- 1 file changed, 160 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 360103982a8..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,13 +2,17 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ + from abc import abstractmethod from functools import cache +import pathlib import astropy.units as u +from astropy.table import Table +import pickle + import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -20,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -28,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -72,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -95,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -235,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -281,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From 010aea6349b647c50c0a0353b1aacd4c58aebb05 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 031/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From 26ff8a4aae421664f912871327335b3dd613b5e2 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 08:51:19 +0200 Subject: [PATCH 032/221] added stats extractor parent component added PlainExtractor based on numpy and scipy functions --- src/ctapipe/calib/camera/extractor.py | 86 +++++++++++++++++++++++++++ src/ctapipe/containers.py | 3 + 2 files changed, 89 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..0140ed5bcb4 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,86 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", +] + + +from abc import abstractmethod + +import numpy as np +import scipy.stats +from traitlets import Int + +from ctapipe.core import TelescopeComponent +from ctapipe.containers import StatisticsContainer + + +class StatisticsExtractor(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__(self, images, trigger_times) -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + images : ndarray + images stored in a numpy array of shape + (n_images, n_channels, n_pix). + trigger_times : ndarray + images stored in a numpy array of shape + (n_images, ) + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(img, time)) + + return stats_list + + def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=np.nanmean(images, axis=0), + median=np.nanmedian(images, axis=0), + std=np.nanstd(images, axis=0), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 9a5b39f4da8..1f2334ba386 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -414,9 +414,12 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics""" + validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") From d142995a203141dd3497d2e2833694bab867712b Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 15:51:47 +0200 Subject: [PATCH 033/221] added stats extractor based on sigma clipping --- src/ctapipe/calib/camera/extractor.py | 67 ++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0140ed5bcb4..654be103f8b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,6 +5,7 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "SigmaClippingExtractor", ] @@ -12,10 +13,14 @@ import numpy as np import scipy.stats -from traitlets import Int +from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core.traits import ( + Int, + List, +) class StatisticsExtractor(TelescopeComponent): @@ -84,3 +89,63 @@ def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: skewness=scipy.stats.skew(images, axis=0), kurtosis=scipy.stats.kurtosis(images, axis=0), ) + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(img, time)) + + return stats_list + + def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + + # mean, median, and std over the sample per pixel + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + std=pixel_std.filled(np.nan), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) + From 0737b7f3f64d8b481a7ebd13358131ee6092db43 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Tue, 30 Apr 2024 16:34:57 +0200 Subject: [PATCH 034/221] added cut of outliers restructured the stats containers --- src/ctapipe/calib/camera/extractor.py | 139 +++++++++++++++------ src/ctapipe/containers.py | 17 ++- src/ctapipe/image/statistics.py | 6 +- src/ctapipe/image/tests/test_statistics.py | 4 +- 4 files changed, 119 insertions(+), 47 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 654be103f8b..6e7ca6aa634 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -24,6 +24,17 @@ class StatisticsExtractor(TelescopeComponent): + + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics @@ -36,19 +47,18 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, images, trigger_times) -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: """ Call the relevant functions to extract the statistics for the particular extractor. Parameters ---------- - images : ndarray - images stored in a numpy array of shape + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). - trigger_times : ndarray - images stored in a numpy array of shape - (n_images, ) + col_name : string + column name in the dl1 table Returns ------- @@ -62,42 +72,60 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - sample_size = Int(2500, help="sample size").tag(config=True) + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: - def __call__(self, dl1_table, col_name="image") -> list: - # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), - mean=np.nanmean(images, axis=0), - median=np.nanmedian(images, axis=0), - std=np.nanstd(images, axis=0), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using astropy's sigma clipping functions """ - sample_size = Int(2500, help="sample size").tag(config=True) - sigma_clipping_max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", @@ -107,8 +135,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - - def __call__(self, dl1_table, col_name="image") -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) @@ -116,17 +143,26 @@ def __call__(self, dl1_table, col_name="image") -> list: # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, + masked_images, + sigma=max_sigma, maxiters=self.sigma_clipping_iterations, cenfunc="mean", axis=0, @@ -137,15 +173,42 @@ def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContaine pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), std=pixel_std.filled(np.nan), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + std_outliers=image_std_outliers.filled(True), ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 1f2334ba386..b8ce24e3973 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", "SchedulingBlockContainer", @@ -412,24 +413,32 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics""" + """Store descriptive statistics of a sequence of images""" validity_start = Field(np.float32(nan), "start") validity_stop = Field(np.float32(nan), "stop") + mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") + median_outliers = Field(np.float32(nan), "median intensity") + std = Field(np.float32(nan), "standard deviation of intensity") + std_outliers = Field(np.float32(nan), "standard deviation intensity") + +class ImageStatisticsContainer(Container): + """Store descriptive image statistics""" + max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") -class IntensityStatisticsContainer(StatisticsContainer): +class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" -class PeakTimeStatisticsContainer(StatisticsContainer): +class PeakTimeStatisticsContainer(ImageStatisticsContainer): default_prefix = "peak_time" diff --git a/src/ctapipe/image/statistics.py b/src/ctapipe/image/statistics.py index bd38f02a377..deb209efd08 100644 --- a/src/ctapipe/image/statistics.py +++ b/src/ctapipe/image/statistics.py @@ -3,7 +3,7 @@ import numpy as np from numba import float32, float64, guvectorize, int64, njit -from ..containers import StatisticsContainer +from ..containers import ImageStatisticsContainer __all__ = [ "arg_n_largest", @@ -88,8 +88,8 @@ def kurtosis(data, mean=None, std=None, fisher=True): def descriptive_statistics( - values, container_class=StatisticsContainer -) -> StatisticsContainer: + values, container_class=ImageStatisticsContainer +) -> ImageStatisticsContainer: """compute intensity statistics of an image""" mean = values.mean() std = values.std() diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 006c9c0dbce..23806705787 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,14 +49,14 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, StatisticsContainer + from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) data = rng.normal(5, 2, 1000) stats = descriptive_statistics(data) - assert isinstance(stats, StatisticsContainer) + assert isinstance(stats, ImageStatisticsContainer) stats = descriptive_statistics(data, container_class=PeakTimeStatisticsContainer) assert isinstance(stats, PeakTimeStatisticsContainer) From 225346b179ec7b46f9eb715b1adbf2d393d07e19 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:10:59 +0200 Subject: [PATCH 035/221] update docs --- src/ctapipe/calib/camera/extractor.py | 4 +++- src/ctapipe/containers.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 6e7ca6aa634..56b8a7f2219 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,7 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) + sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -57,6 +57,8 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> dl1_table : ndarray dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). + masked_pixels_of_sample : ndarray + boolean array of masked pixels that are not available for processing col_name : string column name in the dl1 table diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index b8ce24e3973..5be78775580 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -415,13 +415,13 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") - mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") - median_outliers = Field(np.float32(nan), "median intensity") - std = Field(np.float32(nan), "standard deviation of intensity") - std_outliers = Field(np.float32(nan), "standard deviation intensity") + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + median = Field(np.float32(nan), "Channel-wise and pixel-wise median") + median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") + std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") + std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 74c72bbc985ceab2cd1ccfc02ac0410d3b547b24 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:12:33 +0200 Subject: [PATCH 036/221] formatted with black --- src/ctapipe/calib/camera/extractor.py | 87 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 56b8a7f2219..907f22923d2 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,10 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) + sample_size = Int( + 2500, + help="Size of the sample used for the calculation of the statistical values", + ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -47,7 +50,9 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -68,31 +73,41 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> List of extracted statistics and validity ranges """ + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using numpy and scipy functions """ - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -137,25 +152,34 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median of the event images image_median = np.ma.median(masked_images, axis=-1) @@ -194,15 +218,21 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = ( - np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], - image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = ( - np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) return StatisticsContainer( validity_start=times[0], @@ -213,4 +243,3 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S std=pixel_std.filled(np.nan), std_outliers=image_std_outliers.filled(True), ) - From faad542fab980069fe6c8780f6b40b8577322b62 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:29:10 +0200 Subject: [PATCH 037/221] added pass for __call__ function --- src/ctapipe/calib/camera/extractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 907f22923d2..e4ed0342e1b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -72,6 +72,7 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ + pass class PlainExtractor(StatisticsExtractor): From 9867431b7f3f5545c1f260933d9d8c132c4c25f2 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 7 May 2024 09:30:29 +0200 Subject: [PATCH 038/221] Small commit for prototyping --- src/ctapipe/calib/camera/extractor.py | 46 ++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index e4ed0342e1b..718aa48494b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -6,6 +6,7 @@ "StatisticsExtractor", "PlainExtractor", "SigmaClippingExtractor", + "StarExtractor", ] @@ -14,6 +15,8 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats +from astropy.coordinates import EarthLocation, SkyCoord, Angle +from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -21,6 +24,7 @@ Int, List, ) +from ctapipe.coordinates import EngineeringCameraFrame class StatisticsExtractor(TelescopeComponent): @@ -138,9 +142,49 @@ def _plain_extraction( ) +class StarExtractor(StatisticsExtractor): + """ + Extracts pointing information from a series of variance images + using the startracker functions + """ + + min_star_magnitude = Float( + 0.1, + help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + ).tag(config=True) + + def __init__(): + + def __call__( + self, variance_table, initial_pointing, PSF_model + ): + + def _stars_in_FOV( + self, pointing + ): + + stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] + + for star in stars: + + star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') + star_coords = star_coords.transform_to(camera_frame) + central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( + + def _star_extraction( + self, + ): + camera_frame = EngineeringCameraFrame( + telescope_pointing=current_pointing, + focal_length=self.focal_length, + obstime=time.utc, + + + + class SigmaClippingExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extracts the statistics from a sequence of images using astropy's sigma clipping functions """ From 3a85ffab643414a0f4c0a090301b1e555b5610a1 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 10 May 2024 09:05:01 +0200 Subject: [PATCH 039/221] Removed unneeded functions --- src/ctapipe/calib/camera/extractor.py | 28 +++------------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 718aa48494b..eb245447527 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,8 +5,8 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "StarVarianceExtractor", "SigmaClippingExtractor", - "StarExtractor", ] @@ -15,8 +15,6 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats -from astropy.coordinates import EarthLocation, SkyCoord, Angle -from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -142,7 +140,7 @@ def _plain_extraction( ) -class StarExtractor(StatisticsExtractor): +class StarVarianceExtractor(StatisticsExtractor): """ Extracts pointing information from a series of variance images using the startracker functions @@ -156,29 +154,9 @@ class StarExtractor(StatisticsExtractor): def __init__(): def __call__( - self, variance_table, initial_pointing, PSF_model + self, variance_table, trigger_table, initial_pointing, PSF_model ): - def _stars_in_FOV( - self, pointing - ): - - stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] - - for star in stars: - - star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') - star_coords = star_coords.transform_to(camera_frame) - central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( - - def _star_extraction( - self, - ): - camera_frame = EngineeringCameraFrame( - telescope_pointing=current_pointing, - focal_length=self.focal_length, - obstime=time.utc, - From 69ebd1c38402a8fcc23c0af84b835279a2c56a13 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:41:44 +0200 Subject: [PATCH 040/221] added unit tests --- .../calib/camera/tests/test_extractors.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..5e5f7a617fc --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,51 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import astropy.units as u +from astropy.table import QTable +import numpy as np + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +def test_extractors(example_subarray): + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False + +def test_check_outliers(example_subarray): + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:,0,120] = 120.0 + flatfield_dl1_data[:,1,67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + #check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] == True + assert sigmaclipping_stats_list[0].median_outliers[1][67] == True + assert sigmaclipping_stats_list[1].median_outliers[0][120] == True + assert sigmaclipping_stats_list[1].median_outliers[1][67] == True From 96b2dbbaec2147c4f8cac530bc7f5c1038dd622a Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:50:48 +0200 Subject: [PATCH 041/221] added changelog --- docs/changes/2554.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/2554.feature.rst diff --git a/docs/changes/2554.feature.rst b/docs/changes/2554.feature.rst new file mode 100644 index 00000000000..2e6a6356b3a --- /dev/null +++ b/docs/changes/2554.feature.rst @@ -0,0 +1 @@ +Add API to extract the statistics from a sequence of images. From a0077732e5efa2ca376845ea5d9918fddf5ead81 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 13:54:45 +0200 Subject: [PATCH 042/221] fix lint --- src/ctapipe/calib/camera/extractor.py | 9 --------- src/ctapipe/calib/camera/tests/test_extractors.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index eb245447527..c8a1b3158ce 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -121,9 +121,6 @@ def _plain_extraction( # std over the sample per pixel pixel_std = np.ma.std(masked_images, axis=0) - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - # outliers from median image_median_outliers = np.logical_or( pixel_median < self.image_median_cut_outliers[0], @@ -204,9 +201,6 @@ def _sigmaclipping_extraction( # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - # median of the event images - image_median = np.ma.median(masked_images, axis=-1) - # mean, median, and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( @@ -224,9 +218,6 @@ def _sigmaclipping_extraction( unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - # only warn for values discard in the sigma clipping, not those from before - outliers = unused_values & (~masked_images.mask) - # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 5e5f7a617fc..19b04c017fe 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,7 +22,7 @@ def test_extractors(example_subarray): plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False From 96fe563e9ca43bdca33e9f2ae542ed0ba8ae5848 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 23 May 2024 11:37:46 +0200 Subject: [PATCH 043/221] I altered the class variables to th evariance statistics extractor --- src/ctapipe/calib/camera/extractor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index c8a1b3158ce..70f4a2eb561 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -143,20 +143,23 @@ class StarVarianceExtractor(StatisticsExtractor): using the startracker functions """ - min_star_magnitude = Float( - 0.1, - help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) def __init__(): def __call__( - self, variance_table, trigger_table, initial_pointing, PSF_model + self, variance_table ): - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images From 67d34c4499c10a5076a1a35768e6ab6303de13b5 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 24 May 2024 13:38:36 +0200 Subject: [PATCH 044/221] added a container for mean variance images and fixed docustring --- src/ctapipe/calib/camera/extractor.py | 43 +++++++++++++++++++++++++-- src/ctapipe/containers.py | 9 +++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 70f4a2eb561..3cc51de40cb 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -139,8 +139,9 @@ def _plain_extraction( class StarVarianceExtractor(StatisticsExtractor): """ - Extracts pointing information from a series of variance images - using the startracker functions + Generating average variance images from a set + of variance images for the startracker + pointing calibration """ sigma_clipping_max_sigma = Int( @@ -158,7 +159,43 @@ def __call__( self, variance_table ): - + image_chunks = ( + variance_table["image"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["image"].data), self.sample_size) + ) + + time_chunks = ( + variance_table["trigger_times"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) + ) + + stats_list = [] + + for images, times in zip(image_chunks, time_chunks): + + stats_list.append( + self._sigmaclipping_extraction(images, times) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times + )->StatisticsContainer: + + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan) + ) class SigmaClippingExtractor(StatisticsExtractor): """ diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 5be78775580..0aeff5a97eb 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -423,6 +424,13 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") +class VarianceStatisticsContainer(Container): + """Store descriptive statistics of a sequence of variance images""" + + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + class ImageStatisticsContainer(Container): """Store descriptive image statistics""" @@ -433,7 +441,6 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") - class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From f1429da98732bec84d5962cc083a7c27062cbc90 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 27 May 2024 15:20:53 +0200 Subject: [PATCH 045/221] I changed the container type for the StarVarianceExtractor --- src/ctapipe/calib/camera/extractor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 3cc51de40cb..ffcd561aa87 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -180,7 +180,7 @@ def __call__( def _sigmaclipping_extraction( self, images, times - )->StatisticsContainer: + )->VarianceStatisticsContainer: pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( images, @@ -190,11 +190,10 @@ def _sigmaclipping_extraction( axis=0, ) - return StatisticsContainer( + return VarianceStatisticsContainer( validity_start=times[0], validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan) + mean=pixel_mean.filled(np.nan) ) class SigmaClippingExtractor(StatisticsExtractor): From c9ec02b8f4b677a1d2909720c9b0706229ec5132 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:42:35 +0200 Subject: [PATCH 046/221] fix pylint Remove StarVarianceExtractor since is functionality is featured in the existing Extractors --- src/ctapipe/calib/camera/extractor.py | 89 ++++--------------- .../calib/camera/tests/test_extractors.py | 64 +++++++------ src/ctapipe/containers.py | 7 -- 3 files changed, 53 insertions(+), 107 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index ffcd561aa87..9e9e9947462 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,15 +5,12 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", - "StarVarianceExtractor", "SigmaClippingExtractor", ] - from abc import abstractmethod import numpy as np -import scipy.stats from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent @@ -22,10 +19,9 @@ Int, List, ) -from ctapipe.coordinates import EngineeringCameraFrame - class StatisticsExtractor(TelescopeComponent): + """Base StatisticsExtractor component""" sample_size = Int( 2500, @@ -33,11 +29,13 @@ class StatisticsExtractor(TelescopeComponent): ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", + help="""Interval of accepted image values \\ + (fraction with respect to camera median value)""", ).tag(config=True) image_std_cut_outliers = List( [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", + help="""Interval (number of std) of accepted image standard deviation \\ + around camera median value""", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): @@ -74,7 +72,6 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ - pass class PlainExtractor(StatisticsExtractor): @@ -136,66 +133,6 @@ def _plain_extraction( std=pixel_std.filled(np.nan), ) - -class StarVarianceExtractor(StatisticsExtractor): - """ - Generating average variance images from a set - of variance images for the startracker - pointing calibration - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __init__(): - - def __call__( - self, variance_table - ): - - image_chunks = ( - variance_table["image"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["image"].data), self.sample_size) - ) - - time_chunks = ( - variance_table["trigger_times"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) - ) - - stats_list = [] - - for images, times in zip(image_chunks, time_chunks): - - stats_list.append( - self._sigmaclipping_extraction(images, times) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times - )->VarianceStatisticsContainer: - - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - return VarianceStatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan) - ) - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images @@ -273,18 +210,26 @@ def _sigmaclipping_extraction( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], ) return StatisticsContainer( diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 19b04c017fe..89c375387e3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,7 +2,6 @@ Tests for StatisticsExtractor and related functions """ -import astropy.units as u from astropy.table import QTable import numpy as np @@ -10,42 +9,51 @@ def test_extractors(example_subarray): + """test basic functionality of the StatisticsExtractors""" + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) - + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False - + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + + def test_check_outliers(example_subarray): - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + """test detection ability of outliers""" + + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers - flatfield_dl1_data[:,0,120] = 120.0 - flatfield_dl1_data[:,1,67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - #check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] == True - assert sigmaclipping_stats_list[0].median_outliers[1][67] == True - assert sigmaclipping_stats_list[1].median_outliers[0][120] == True - assert sigmaclipping_stats_list[1].median_outliers[1][67] == True + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] is True + assert sigmaclipping_stats_list[0].median_outliers[1][67] is True + assert sigmaclipping_stats_list[1].median_outliers[0][120] is True + assert sigmaclipping_stats_list[1].median_outliers[1][67] is True diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 0aeff5a97eb..68685a26623 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,7 +57,6 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", - "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -424,12 +423,6 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") -class VarianceStatisticsContainer(Container): - """Store descriptive statistics of a sequence of variance images""" - - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 444e37055c82efa5665bc86632b565125bb1832f Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:45:29 +0200 Subject: [PATCH 047/221] change __call__() to _extract() --- src/ctapipe/calib/camera/extractor.py | 6 +++--- src/ctapipe/calib/camera/tests/test_extractors.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 9e9e9947462..eaff6c714c5 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -80,7 +80,7 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: @@ -148,7 +148,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 89c375387e3..d5f082762ed 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,8 +22,8 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -50,7 +50,7 @@ def test_check_outliers(example_subarray): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From 1c4cadbf5fe6b11f680f10444ea3c44b5a3b5d57 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:59:36 +0200 Subject: [PATCH 048/221] minor renaming --- src/ctapipe/calib/camera/extractor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index eaff6c714c5..0c23caebb00 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -60,7 +60,7 @@ def _extract( Parameters ---------- dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape + dl1 table with images and timestamps stored in a numpy array of shape (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing @@ -139,11 +139,11 @@ class SigmaClippingExtractor(StatisticsExtractor): using astropy's sigma clipping functions """ - sigma_clipping_max_sigma = Int( + max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", ).tag(config=True) - sigma_clipping_iterations = Int( + iterations = Int( default_value=5, help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) @@ -178,11 +178,10 @@ def _sigmaclipping_extraction( masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, + sigma=self.max_sigma, + maxiters=self.iterations, cenfunc="mean", axis=0, ) @@ -192,7 +191,7 @@ def _sigmaclipping_extraction( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 0ebb464f5ece32f71c5f150d91ed0ffb6fb776a9 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:15:53 +0200 Subject: [PATCH 049/221] use pytest.fixture for Extractors --- .../calib/camera/tests/test_extractors.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d5f082762ed..d363ee24ff0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -7,14 +7,21 @@ from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor +@pytest.fixture + def test_plainextractor(example_subarray): + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -def test_extractors(example_subarray): +@pytest.fixture + def test_sigmaclippingextractor(example_subarray): + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" - plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) @@ -22,8 +29,10 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + dl1_table=flatfield_dl1_table + ) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -38,19 +47,18 @@ def test_extractors(example_subarray): assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False -def test_check_outliers(example_subarray): +def test_check_outliers(test_sigmaclippingextractor): """test detection ability of outliers""" - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract( + dl1_table=flatfield_dl1_table + ) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From bbbd8911067965a28474f3ed29c8b72ad76d7350 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:59:57 +0200 Subject: [PATCH 050/221] reduce duplicated code of the call function --- src/ctapipe/calib/camera/extractor.py | 52 ++++++------------- .../calib/camera/tests/test_extractors.py | 30 ++++++----- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0c23caebb00..d060d620508 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -49,8 +49,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - @abstractmethod - def _extract( + def __call__( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -73,17 +72,6 @@ def _extract( List of extracted statistics and validity ranges """ - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] @@ -98,11 +86,23 @@ def _extract( stats_list = [] for images, times in zip(image_chunks, time_chunks): stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) + self._extract(images, times, masked_pixels_of_sample) ) return stats_list - def _plain_extraction( + @abstractmethod + def _extract( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + pass + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def _extract( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: @@ -149,28 +149,6 @@ class SigmaClippingExtractor(StatisticsExtractor): ).tag(config=True) def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d363ee24ff0..06107a6e7b7 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -4,20 +4,22 @@ from astropy.table import QTable import numpy as np - +import pytest from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor -@pytest.fixture - def test_plainextractor(example_subarray): - return PlainExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -@pytest.fixture - def test_sigmaclippingextractor(example_subarray): - return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -29,8 +31,8 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) @@ -56,7 +58,7 @@ def test_check_outliers(test_sigmaclippingextractor): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract( + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) From 5deef740c1af2ee119a5b0188ce3909bd7ab113f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 051/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 223 ++++++++++++++++++++++++- src/ctapipe/image/psf_model.py | 80 +++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index baf3d2f1057..81cacedc432 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,24 +2,33 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Integer, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource -__all__ = ["CameraCalibrator"] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -47,6 +56,218 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) + + @abstractmethod + def __call__(self, data_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated + tel_id : int + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties + """ + + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) + + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): + """ + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + pass + + +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + PSFModel_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="PlainExtractor", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + self.psf = PSFModel.from_name( + self.PSFModel_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, url, tel_id): + if self._check_req_data(url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + with EventSource(url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + def _calibrate_varimages(self, varimages, gain): + pass + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..7af526e6c21 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,80 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + +from ctapipe.core import TelescopeComponent + + +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base Component to describe image distortion due to the optics of the different cameras. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) From f09bbe112aaa498d69a55bf1b1c78e5f4dd32c14 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 052/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 81cacedc432..cd7ad324e1f 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -266,6 +266,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From b8f0a4fbe6fe20d29b6a31812030ddd1da9f155e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 053/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index cd7ad324e1f..0c48113e4ae 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -111,21 +111,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -208,7 +220,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -224,7 +236,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -264,7 +276,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From 39fca21ebf0b2cb75c37d3c2f0faf83e4b3c6821 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 054/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 0c48113e4ae..f921b2d97fb 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -276,9 +276,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From b11463890027ecf67f6e5715d00313312d9f791d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 12:02:41 +0200 Subject: [PATCH 055/221] edit description of StatisticsContainer --- src/ctapipe/containers.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 68685a26623..f61e429417d 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -415,13 +415,33 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") - median = Field(np.float32(nan), "Channel-wise and pixel-wise median") - median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") - std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") - std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") + extraction_start = Field(np.float32(nan), "start of the extraction sequence") + extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + mean = Field( + None, + "mean of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median = Field( + None, + "median of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median_outliers = Field( + None, + "outliers from the median distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) + std = Field( + None, + "standard deviation of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + std_outliers = Field( + None, + "outliers from the standard deviation distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) class ImageStatisticsContainer(Container): @@ -434,6 +454,7 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") + class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From 5d18f61d49635cd3ffdc6a06edf7d7f162b67698 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 13:12:54 +0200 Subject: [PATCH 056/221] added feature to shift the extraction sequence allow overlapping extraction sequences --- src/ctapipe/calib/camera/extractor.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index d060d620508..f4ddb03d4bd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,11 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" + self, + dl1_table, + masked_pixels_of_sample=None, + sample_shift=None, + col_name="image", ) -> list: """ Call the relevant functions to extract the statistics @@ -63,6 +67,8 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing + sample_shift : int + number of samples to shift the extraction sequence col_name : string column name in the dl1 table @@ -72,14 +78,19 @@ def __call__( List of extracted statistics and validity ranges """ + # If no sample_shift is provided, the sample_shift is set to self.sample_size + # meaning that the samples are not overlapping. + if sample_shift is None: + sample_shift = self.sample_size + # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) + for i in range(0, len(dl1_table[col_name].data), sample_shift) ) time_chunks = ( dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) + for i in range(0, len(dl1_table["time"]), sample_shift) ) # Calculate the statistics from a sequence of images @@ -169,7 +180,9 @@ def _extract( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > ( + self.max_sigma * pixel_std + ) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 4df8c94739f4577ac30452ceb94d5aabd5ae60bd Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 14:23:53 +0200 Subject: [PATCH 057/221] fix boundary case for the last chunk renaming to chunk(s) and chunk_size and _shift added test for chunk_shift and boundary case --- src/ctapipe/calib/camera/extractor.py | 72 ++++++++++--------- .../calib/camera/tests/test_extractors.py | 21 +++++- src/ctapipe/containers.py | 6 +- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index f4ddb03d4bd..86d9c1345fd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,5 +1,5 @@ """ -Extraction algorithms to compute the statistics from a sequence of images +Extraction algorithms to compute the statistics from a chunk of images """ __all__ = [ @@ -23,9 +23,9 @@ class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" - sample_size = Int( + chunk_size = Int( 2500, - help="Size of the sample used for the calculation of the statistical values", + help="Size of the chunk used for the calculation of the statistical values", ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], @@ -41,7 +41,7 @@ class StatisticsExtractor(TelescopeComponent): def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). + from a chunk of charges and pulse times (images). Parameters ---------- @@ -53,7 +53,7 @@ def __call__( self, dl1_table, masked_pixels_of_sample=None, - sample_shift=None, + chunk_shift=None, col_name="image", ) -> list: """ @@ -67,38 +67,44 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing - sample_shift : int - number of samples to shift the extraction sequence + chunk_shift : int + number of samples to shift the extraction chunk col_name : string column name in the dl1 table Returns ------- List StatisticsContainer: - List of extracted statistics and validity ranges + List of extracted statistics and extraction chunks """ - # If no sample_shift is provided, the sample_shift is set to self.sample_size - # meaning that the samples are not overlapping. - if sample_shift is None: - sample_shift = self.sample_size - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), sample_shift) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), sample_shift) - ) - - # Calculate the statistics from a sequence of images + # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size + # meaning that the extraction chunks are not overlapping. + if chunk_shift is None: + chunk_shift = self.chunk_size + + # Function to split table data into appropriated chunks + def _get_chunks(col_name): + return [ + ( + dl1_table[col_name].data[i : i + self.chunk_size] + if i + self.chunk_size <= len(dl1_table[col_name]) + else dl1_table[col_name].data[ + len(dl1_table[col_name].data) + - self.chunk_size : len(dl1_table[col_name].data) + ] + ) + for i in range(0, len(dl1_table[col_name].data), chunk_shift) + ] + + # Get the chunks for the timestamps and selected column name + time_chunks = _get_chunks("time") + image_chunks = _get_chunks(col_name) + + # Calculate the statistics from a chunk of images stats_list = [] for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._extract(images, times, masked_pixels_of_sample) - ) + stats_list.append(self._extract(images, times, masked_pixels_of_sample)) return stats_list @abstractmethod @@ -109,7 +115,7 @@ def _extract( class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extractor the statistics from a chunk of images using numpy and scipy functions """ @@ -136,8 +142,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), @@ -146,7 +152,7 @@ def _extract( class SigmaClippingExtractor(StatisticsExtractor): """ - Extracts the statistics from a sequence of images + Extracts the statistics from a chunk of images using astropy's sigma clipping functions """ @@ -223,8 +229,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 06107a6e7b7..40efd4f2fc3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -11,14 +11,14 @@ def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" return PlainExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): @@ -67,3 +67,20 @@ def test_check_outliers(test_sigmaclippingextractor): assert sigmaclipping_stats_list[0].median_outliers[1][67] is True assert sigmaclipping_stats_list[1].median_outliers[0][120] is True assert sigmaclipping_stats_list[1].median_outliers[1][67] is True + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, + chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 + diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index f61e429417d..dcd70255519 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -413,10 +413,10 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a sequence of images""" + """Store descriptive statistics of a chunk of images""" - extraction_start = Field(np.float32(nan), "start of the extraction sequence") - extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + extraction_start = Field(np.float32(nan), "start of the extraction chunk") + extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") mean = Field( None, "mean of a pixel-wise quantity for each channel" From 86eb15c0304a4dcddc779311f48d087131f4803f Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:13:27 +0200 Subject: [PATCH 058/221] fix tests --- .../calib/camera/tests/test_extractors.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 40efd4f2fc3..a83c93fd1c0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,24 +2,24 @@ Tests for StatisticsExtractor and related functions """ -from astropy.table import QTable import numpy as np import pytest +from astropy.table import QTable + from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + @pytest.fixture(name="test_plainextractor") def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" - return PlainExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" - return SigmaClippingExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -36,17 +36,17 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): dl1_table=flatfield_dl1_table ) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) def test_check_outliers(test_sigmaclippingextractor): @@ -63,11 +63,11 @@ def test_check_outliers(test_sigmaclippingextractor): ) # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] is True - assert sigmaclipping_stats_list[0].median_outliers[1][67] is True - assert sigmaclipping_stats_list[1].median_outliers[0][120] is True - assert sigmaclipping_stats_list[1].median_outliers[1][67] is True - + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + def test_check_chunk_shift(test_sigmaclippingextractor): """test the chunk shift option and the boundary case for the last chunk""" @@ -77,10 +77,8 @@ def test_check_chunk_shift(test_sigmaclippingextractor): # insert outliers flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, - chunk_shift=2000 + dl1_table=flatfield_dl1_table, chunk_shift=2000 ) # check if three chunks are used for the extraction assert len(sigmaclipping_stats_list) == 3 - From 62006539acb7bc31953c3c767fe8cf8741debfaa Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:26:03 +0200 Subject: [PATCH 059/221] fix ruff --- src/ctapipe/calib/camera/extractor.py | 40 +++++++++------------- src/ctapipe/image/tests/test_statistics.py | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 86d9c1345fd..4c8f49d1f38 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -13,13 +13,14 @@ import numpy as np from astropy.stats import sigma_clipped_stats -from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Int, List, ) + class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" @@ -90,8 +91,9 @@ def _get_chunks(col_name): dl1_table[col_name].data[i : i + self.chunk_size] if i + self.chunk_size <= len(dl1_table[col_name]) else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - - self.chunk_size : len(dl1_table[col_name].data) + len(dl1_table[col_name].data) - self.chunk_size : len( + dl1_table[col_name].data + ) ] ) for i in range(0, len(dl1_table[col_name].data), chunk_shift) @@ -108,21 +110,17 @@ def _get_chunks(col_name): return stats_list @abstractmethod - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: pass + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a chunk of images using numpy and scipy functions """ - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -150,6 +148,7 @@ def _extract( std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a chunk of images @@ -165,10 +164,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -206,25 +202,21 @@ def _extract( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], ) diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 23806705787..4403e05ca0a 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer + from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From 4f604c50245341889698767ca0568d6ee5474f6a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 060/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index f921b2d97fb..360103982a8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -26,7 +26,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -270,6 +270,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -294,7 +299,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From 6dfce150cae5e9c0cb1135930d384372eb275565 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 061/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 227 +++++++++++++++++-------- 1 file changed, 160 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 360103982a8..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,13 +2,17 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ + from abc import abstractmethod from functools import cache +import pathlib import astropy.units as u +from astropy.table import Table +import pickle + import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -20,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -28,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -72,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -95,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -235,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -281,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From 8e873d3c1f60ccfd3f9ffb6e60cf822c15dbc1b3 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 062/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From f7d3223e17abb3dc967ba93acad9c02090563a31 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 7 Aug 2024 21:18:05 +0200 Subject: [PATCH 063/221] solved merge conflicts --- src/ctapipe/calib/camera/extractor.py | 214 ++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..6e7ca6aa634 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,214 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", + "SigmaClippingExtractor", +] + + +from abc import abstractmethod + +import numpy as np +import scipy.stats +from astropy.stats import sigma_clipped_stats + +from ctapipe.core import TelescopeComponent +from ctapipe.containers import StatisticsContainer +from ctapipe.core.traits import ( + Int, + List, +) + + +class StatisticsExtractor(TelescopeComponent): + + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape + (n_images, n_channels, n_pix). + col_name : string + column name in the dl1 table + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + return stats_list + + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + ) + + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + return stats_list + + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) + + # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + masked_images, + sigma=max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + std_outliers=image_std_outliers.filled(True), + ) + From 91812daf69e73423c0b4f426c9e630403b8b48d3 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 064/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 223 ++++++++++++++++++++++++- src/ctapipe/image/psf_model.py | 80 +++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index baf3d2f1057..81cacedc432 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,24 +2,33 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Integer, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource -__all__ = ["CameraCalibrator"] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -47,6 +56,218 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) + + @abstractmethod + def __call__(self, data_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated + tel_id : int + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties + """ + + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) + + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): + """ + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + pass + + +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + PSFModel_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="PlainExtractor", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + self.psf = PSFModel.from_name( + self.PSFModel_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, url, tel_id): + if self._check_req_data(url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + with EventSource(url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + def _calibrate_varimages(self, varimages, gain): + pass + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..7af526e6c21 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,80 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + +from ctapipe.core import TelescopeComponent + + +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base Component to describe image distortion due to the optics of the different cameras. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) From 326c2028d5b3879f2219148f38a622d914e8f82d Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 065/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 81cacedc432..cd7ad324e1f 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -266,6 +266,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From fa029f736041e5c7920d0d986fc6b5874a0e7a46 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 066/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index cd7ad324e1f..0c48113e4ae 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -111,21 +111,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -208,7 +220,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -224,7 +236,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -264,7 +276,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From 7c90e0ffab567c6fbc3ac36d68f649235282a2f6 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 067/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 0c48113e4ae..f921b2d97fb 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -276,9 +276,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From 8c44a42d3a5887fd9d8f5e71d5346ef13041d092 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 068/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index f921b2d97fb..360103982a8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -26,7 +26,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -270,6 +270,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -294,7 +299,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From bff611a39406b6868f8c3e89a154bca8d4110947 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 069/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 227 +++++++++++++++++-------- 1 file changed, 160 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 360103982a8..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,13 +2,17 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ + from abc import abstractmethod from functools import cache +import pathlib import astropy.units as u +from astropy.table import Table +import pickle + import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -20,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -28,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -72,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -95,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -235,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -281,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From d85dd5081e4cf99adc240ad0afc2a5837584e585 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 070/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From 9192790930b59c4d39e501220f96edc60d0d354b Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 12:15:10 +0200 Subject: [PATCH 071/221] Removed Pointing Calculator --- src/ctapipe/calib/camera/calibrator.py | 164 +------------------------ 1 file changed, 5 insertions(+), 159 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ae0dfdecbbe..07cd295d06c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -3,16 +3,12 @@ calibration and image extraction, as well as supporting algorithms. """ +import pickle from abc import abstractmethod from functools import cache -import pathlib import astropy.units as u -from astropy.table import Table -import pickle - import numpy as np -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -21,23 +17,19 @@ from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, Float, Int, - Integer, Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, TableLoader +from ctapipe.io import TableLoader __all__ = [ "CalibrationCalculator", "TwoPassStatisticsCalculator", - "PointingCalculator", "CameraCalibrator", ] @@ -146,7 +138,7 @@ class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + faulty_pixels_threshold = Float( 0.1, help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", @@ -155,14 +147,13 @@ class TwoPassStatisticsCalculator(CalibrationCalculator): 100, help="Number of samples to shift the extraction chunk for the calculation of the statistical values", ).tag(config=True) - + def __call__( self, input_url, tel_id, col_name="image", ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( @@ -187,7 +178,6 @@ def __call__( stats_list = [] faultless_previous_chunk = False for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk if faultless_previous_chunk: stats_list.append(stats_list_firstpass[chunk_nr - 1]) @@ -215,7 +205,7 @@ def __call__( ) # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously caluclated start and stop. + # Slice the dl1 table according to the previously calculated start and stop. dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. @@ -237,7 +227,6 @@ def __call__( with open(self.output_path, "wb") as f: pickle.dump(stats_list, f) - def _get_slice_range( self, chunk_nr, @@ -260,149 +249,6 @@ def _get_slice_range( return slice_start, slice_stop -class PointingCalculator(CalibrationCalculator): - """ - Component to calculate pointing corrections from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - telescope_location: dict - The location of the telescope for which the pointing correction is to be calculated - """ - - telescope_location = Dict( - {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, - help="Telescope location, longitude and latitude should be expressed in deg, " - "elevation - in meters", - ).tag(config=True) - - min_star_prominence = Integer( - 3, - help="Minimal star prominence over the background in terms of " - "NSB variance std deviations", - ).tag(config=True) - - max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" - ).tag(config=True) - - psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), - default_value="PlainExtractor", - help="Name of the PSFModel Subclass to be used.", - ).tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - **kwargs, - ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - # TODO: Currently not in the dependency list of ctapipe - from astroquery.vizier import Vizier - - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self - ) - - self.location = EarthLocation( - lon=self.telescope_location["longitude"] * u.deg, - lat=self.telescope_location["latitude"] * u.deg, - height=self.telescope_location["elevation"] * u.m, - ) - - def __call__(self, url, tel_id): - if self._check_req_data(url, tel_id, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - self.tel_id = tel_id - - with EventSource(url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) - - with TableLoader(url) as loader: - loader.read_telescope_events_by_id( - telescopes=[tel_id], dl1_parameters=True, observation_info=True - ) - - stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] - - stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, gain): - # So, here i need to match up the validity periods of the relative gain to the variance images - gain_to_variance = np.zeros( - len(var_images) - ) # this array will map the gain values to accumulated variance images - - for i in np.arange( - 1, len(var_images) - ): # the first pairing is 0 -> 0, so start at 1 - for j in np.arange(len(gain), 0): - if var_images[i].validity_start > gain[j].validity_start or j == len( - var_images - ): - gain_to_variance[i] = j - break - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square( - gain[gain_to_variance[i]] - ), # Here i will need to adjust the code based on how the containers for gain will work - ) - - return var_images - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill From b729f8294cd558a5072bf443f4da4ac63fb5a178 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:31:20 +0200 Subject: [PATCH 072/221] Copying over code for interpolators and pointing calculators --- src/ctapipe/calib/camera/calibrator.py | 364 +++++++++++++++++++++- src/ctapipe/image/psf_model.py | 95 ++++++ src/ctapipe/io/interpolation.py | 345 ++++++++++++++++++++ src/ctapipe/io/tests/test_interpolator.py | 179 +++++++++++ 4 files changed, 982 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py create mode 100644 src/ctapipe/io/interpolation.py create mode 100644 src/ctapipe/io/tests/test_interpolator.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index baf3d2f1057..6c72ff5fa6a 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,24 +2,42 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ + +import pickle +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +import Vizier +from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Int, + Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = ["CameraCalibrator"] +__all__ = [ + "CalibrationCalculator", + "TwoPassStatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -47,6 +65,350 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + output_path = Path(help="output filename").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_extractor=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class TwoPassStatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) + + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) + + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > self.faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously calculated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + def _get_slice_range( + self, + chunk_nr, + chunk_size, + faultless_previous_chunk, + last_chunk, + last_element, + ): + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + self.chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + self.chunk_shift + ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + + return slice_start, slice_stop + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + psf_model_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="ComaModel", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + # TODO: Currently not in the dependency list of ctapipe + + self.psf = PSFModel.from_name( + self.pas_model_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, input_url, tel_id): + if self._check_req_data(input_url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + # first get thecamera geometry and pointing for the file and determine what stars we should see + + with EventSource(input_url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + + # get the time and images from the data + + variance_images = dl1_table["variance_image"] + + time = dl1_table["time"] + + # now calibrate the images + + variance_images = self._calibrate_var_images( + self, variance_images, time, input_url + ) + + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + + def _calibrate_var_images(self, var_images, time, calibration_file): + # So i need to use the interpolator classes to read the calibration data + relative_gains = FlatFieldInterpolator( + calibration_file + ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, + np.square(relative_gains(time[i])), + ) + + return var_images + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..458070b8145 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,95 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + + +class PSFModel: + def __init__(self, **kwargs): + """ + Base component to describe image distortion due to the optics of the different cameras. + """ + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py new file mode 100644 index 00000000000..3b792c2107d --- /dev/null +++ b/src/ctapipe/io/interpolation.py @@ -0,0 +1,345 @@ +from abc import ABCMeta, abstractmethod +from typing import Any + +import astropy.units as u +import numpy as np +import tables +from astropy.time import Time +from scipy.interpolate import interp1d + +from ctapipe.core import Component, traits + +from .astropy_helpers import read_table + + +class StepFunction: + + """ + Step function Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the closest previous + point given. + + Parameters + ---------- + values : None | np.array + Numpy array of the data that is to be interpolated. + The first dimension needs to be an index over time + times : None | np.array + Time values over which data are to be interpolated + need to be sorted and have same length as first dimension of values + """ + + def __init__( + self, + times, + values, + bounds_error=True, + fill_value="extrapolate", + assume_sorted=True, + copy=False, + ): + self.values = values + self.times = times + self.bounds_error = bounds_error + self.fill_value = fill_value + + def __call__(self, point): + if point < self.times[0]: + if self.bounds_error: + raise ValueError("below the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[0] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + else: + i = np.searchsorted(self.times, point, side="left") + return self.values[i - 1] + + +class Interpolator(Component, metaclass=ABCMeta): + """ + Interpolator parent class. + + Parameters + ---------- + h5file : None | tables.File + A open hdf5 file with read access. + """ + + bounds_error = traits.Bool( + default_value=True, + help="If true, raises an exception when trying to extrapolate out of the given table", + ).tag(config=True) + + extrapolate = traits.Bool( + help="If bounds_error is False, this flag will specify whether values outside" + "the available values are filled with nan (False) or extrapolated (True).", + default_value=False, + ).tag(config=True) + + telescope_data_group = None + required_columns = set() + expected_units = {} + + def __init__(self, h5file=None, **kwargs): + super().__init__(**kwargs) + + if h5file is not None and not isinstance(h5file, tables.File): + raise TypeError("h5file must be a tables.File") + self.h5file = h5file + + self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) + if self.bounds_error: + self.interp_options["bounds_error"] = True + elif self.extrapolate: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = "extrapolate" + else: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = np.nan + + self._interpolators = {} + + @abstractmethod + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + This method reads input tables and creates instances of the needed interpolators + to be added to _interpolators. The first index of _interpolators needs to be + tel_id, the second needs to be the name of the parameter that is to be interpolated + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are always ``time`` as ``Time`` column and + other columns for the data that is to be interpolated + """ + + pass + + def _check_tables(self, input_table): + missing = self.required_columns - set(input_table.colnames) + if len(missing) > 0: + raise ValueError(f"Table is missing required column(s): {missing}") + for col in self.expected_units: + unit = input_table[col].unit + if unit is None: + if self.expected_units[col] is not None: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + elif not self.expected_units[col].is_equivalent(unit): + if self.expected_units[col] is None: + raise ValueError(f"{col} must have units compatible with 'None'") + else: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + + def _check_interpolators(self, tel_id): + if tel_id not in self._interpolators: + if self.h5file is not None: + self._read_parameter_table(tel_id) # might need to be removed + else: + raise KeyError(f"No table available for tel_id {tel_id}") + + def _read_parameter_table(self, tel_id): + input_table = read_table( + self.h5file, + f"{self.telescope_data_group}/tel_{tel_id:03d}", + ) + self.add_table(tel_id, input_table) + + +class PointingInterpolator(Interpolator): + """ + Interpolator for pointing and pointing correction data + """ + + telescope_data_group = "/dl0/monitoring/telescope/pointing" + required_columns = frozenset(["time", "azimuth", "altitude"]) + expected_units = {"azimuth": u.rad, "altitude": u.rad} + + def __call__(self, tel_id, time): + """ + Interpolate alt/az for given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the pointing + + Returns + ------- + altitude : astropy.units.Quantity[deg] + interpolated altitude angle + azimuth : astropy.units.Quantity[deg] + interpolated azimuth angle + """ + + self._check_interpolators(tel_id) + + mjd = time.tai.mjd + az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) + alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) + return alt, az + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` + as quantity columns for pointing and pointing correction data. + """ + + self._check_tables(input_table) + + if not isinstance(input_table["time"], Time): + raise TypeError("'time' column of pointing table must be astropy.time.Time") + + input_table = input_table.copy() + input_table.sort("time") + + az = input_table["azimuth"].quantity.to_value(u.rad) + # prepare azimuth for interpolation by "unwrapping": i.e. turning + # [359, 1] into [359, 361]. This assumes that if we get values like + # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees + # the other way around. This should be true for all telescopes given + # the sampling speed of pointing values and their maximum movement speed. + # No telescope can turn more than 180° in 2 seconds. + az = np.unwrap(az) + alt = input_table["altitude"].quantity.to_value(u.rad) + mjd = input_table["time"].tai.mjd + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) + self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) + + +class FlatFieldInterpolator(Interpolator): + """ + Interpolator for flatfield data + """ + + telescope_data_group = "dl1/calibration/gain" # TBD + required_columns = frozenset(["time", "gain"]) # TBD + expected_units = {"gain": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate flatfield data for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + ffield : array [float] + interpolated flatfield data + """ + + self._check_interpolators(tel_id) + + ffield = self._interpolators[tel_id]["gain"](time) + return ffield + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "gain" + for the flatfield data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + gain = input_table["gain"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["gain"] = StepFunction( + time, gain, **self.interp_options + ) + + +class PedestalInterpolator(Interpolator): + """ + Interpolator for Pedestal data + """ + + telescope_data_group = "dl1/calibration/pedestal" # TBD + required_columns = frozenset(["time", "pedestal"]) # TBD + expected_units = {"pedestal": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate pedestal or gain for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + pedestal : array [float] + interpolated pedestal values + """ + + self._check_interpolators(tel_id) + + pedestal = self._interpolators[tel_id]["pedestal"](time) + return pedestal + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "pedestal" + for the pedestal data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + pedestal = input_table["pedestal"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["pedestal"] = StepFunction( + time, pedestal, **self.interp_options + ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py new file mode 100644 index 00000000000..930e1e7d73c --- /dev/null +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -0,0 +1,179 @@ +import astropy.units as u +import numpy as np +import pytest +import tables +from astropy.table import Table +from astropy.time import Time + +from ctapipe.io.interpolation import ( + FlatFieldInterpolator, + PedestalInterpolator, + PointingInterpolator, +) + +t0 = Time("2022-01-01T00:00:00") + + +def test_azimuth_switchover(): + """Test pointing interpolation""" + + table = Table( + { + "time": t0 + [0, 1, 2] * u.s, + "azimuth": [359, 1, 3] * u.deg, + "altitude": [60, 61, 62] * u.deg, + }, + ) + + interpolator = PointingInterpolator() + interpolator.add_table(1, table) + + alt, az = interpolator(tel_id=1, time=t0 + 0.5 * u.s) + assert u.isclose(az, 360 * u.deg) + assert u.isclose(alt, 60.5 * u.deg) + + +def test_invalid_input(): + """Test invalid pointing tables raise nice errors""" + + wrong_time = Table( + { + "time": [1, 2, 3] * u.s, + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3] * u.deg, + } + ) + + interpolator = PointingInterpolator() + with pytest.raises(TypeError, match="astropy.time.Time"): + interpolator.add_table(1, wrong_time) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.m, + "altitude": [1, 2, 3] * u.deg, + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3], + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + +def test_hdf5(tmp_path): + """Test writing interpolated data to file""" + from ctapipe.io import write_table + + table = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + path = tmp_path / "pointing.h5" + write_table(table, path, "/dl0/monitoring/telescope/pointing/tel_001") + with tables.open_file(path) as h5file: + interpolator = PointingInterpolator(h5file) + alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + +def test_bounds(): + """Test invalid pointing tables raise nice errors""" + + table_pointing = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + table_pedestal = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + table_flatfield = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + interpolator_pointing = PointingInterpolator() + interpolator_pedestal = PedestalInterpolator() + interpolator_flatfield = FlatFieldInterpolator() + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + error_message = "below the interpolation range" + + with pytest.raises(ValueError, match=error_message): + interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) + + with pytest.raises(ValueError, match=error_message): + interpolator_pedestal(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match=error_message): + interpolator_flatfield(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match="above the interpolation range"): + interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + pedestal = interpolator_pedestal(tel_id=1, time=1.0) + assert all(pedestal == table_pedestal["pedestal"][0]) + flatfield = interpolator_flatfield(tel_id=1, time=1.0) + assert all(flatfield == table_flatfield["gain"][0]) + with pytest.raises(KeyError): + interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) + with pytest.raises(KeyError): + interpolator_pedestal(tel_id=2, time=1.0) + with pytest.raises(KeyError): + interpolator_flatfield(tel_id=2, time=1.0) + + interpolator_pointing = PointingInterpolator(bounds_error=False) + interpolator_pedestal = PedestalInterpolator(bounds_error=False) + interpolator_flatfield = FlatFieldInterpolator(bounds_error=False) + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + for dt in (-0.1, 10.1) * u.s: + alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) + assert np.isnan(alt.value) + assert np.isnan(az.value) + + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) + interpolator_pointing.add_table(1, table_pointing) + alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) + assert u.isclose(alt, 71 * u.deg) + assert u.isclose(az, -1 * u.deg) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 11 * u.s) + assert u.isclose(alt, 59 * u.deg) + assert u.isclose(az, 11 * u.deg) From aecc5dc1558b608095b323d4208efd710e32fa1a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:35:09 +0200 Subject: [PATCH 073/221] Removing PointingCalculator, PSF model and interpolators --- src/ctapipe/image/psf_model.py | 95 ------------- src/ctapipe/io/interpolation.py | 163 ---------------------- src/ctapipe/io/tests/test_interpolator.py | 45 ------ 3 files changed, 303 deletions(-) delete mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py deleted file mode 100644 index 458070b8145..00000000000 --- a/src/ctapipe/image/psf_model.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Models for the Point Spread Functions of the different telescopes -""" - -__all__ = ["PSFModel", "ComaModel"] - -from abc import abstractmethod - -import numpy as np -from scipy.stats import laplace, laplace_asymmetric -from traitlets import List - - -class PSFModel: - def __init__(self, **kwargs): - """ - Base component to describe image distortion due to the optics of the different cameras. - """ - - @classmethod - def from_name(cls, name, **kwargs): - """ - Obtain an instance of a subclass via its name - - Parameters - ---------- - name : str - Name of the subclass to obtain - - Returns - ------- - Instance - Instance of subclass to this class - """ - requested_subclass = cls.non_abstract_subclasses()[name] - return requested_subclass(**kwargs) - - @abstractmethod - def pdf(self, *args): - pass - - @abstractmethod - def update_model_parameters(self, *args): - pass - - -class ComaModel(PSFModel): - """ - PSF model, describing pure coma aberrations PSF effect - """ - - asymmetry_params = List( - default_value=[0.49244797, 9.23573115, 0.15216096], - help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", - ).tag(config=True) - radial_scale_params = List( - default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], - help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", - ).tag(config=True) - az_scale_params = List( - default_value=[0.24271557, 7.5511501, 0.02037972], - help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", - ).tag(config=True) - - def k_func(self, x): - return ( - 1 - - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) - - self.asymmetry_params[2] * x - ) - - def sr_func(self, x): - return ( - self.radial_scale_params[0] - - self.radial_scale_params[1] * x - + self.radial_scale_params[2] * x**2 - - self.radial_scale_params[3] * x**3 - ) - - def sf_func(self, x): - return self.az_scale_params[0] * np.exp( - -self.az_scale_params[1] * x - ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) - - def pdf(self, r, f): - return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( - f, *self.azimuthal_pdf_params - ) - - def update_model_parameters(self, r, f): - k = self.k_func(r) - sr = self.sr_func(r) - sf = self.sf_func(r) - self.radial_pdf_params = (k, r, sr) - self.azimuthal_pdf_params = (f, sf) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py index 3b792c2107d..82de9f0bd19 100644 --- a/src/ctapipe/io/interpolation.py +++ b/src/ctapipe/io/interpolation.py @@ -12,55 +12,6 @@ from .astropy_helpers import read_table -class StepFunction: - - """ - Step function Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the closest previous - point given. - - Parameters - ---------- - values : None | np.array - Numpy array of the data that is to be interpolated. - The first dimension needs to be an index over time - times : None | np.array - Time values over which data are to be interpolated - need to be sorted and have same length as first dimension of values - """ - - def __init__( - self, - times, - values, - bounds_error=True, - fill_value="extrapolate", - assume_sorted=True, - copy=False, - ): - self.values = values - self.times = times - self.bounds_error = bounds_error - self.fill_value = fill_value - - def __call__(self, point): - if point < self.times[0]: - if self.bounds_error: - raise ValueError("below the interpolation range") - - if self.fill_value == "extrapolate": - return self.values[0] - - else: - a = np.empty(self.values[0].shape) - a[:] = np.nan - return a - - else: - i = np.searchsorted(self.times, point, side="left") - return self.values[i - 1] - - class Interpolator(Component, metaclass=ABCMeta): """ Interpolator parent class. @@ -229,117 +180,3 @@ def add_table(self, tel_id, input_table): self._interpolators[tel_id] = {} self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) - - -class FlatFieldInterpolator(Interpolator): - """ - Interpolator for flatfield data - """ - - telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["time", "gain"]) # TBD - expected_units = {"gain": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate flatfield data for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - ffield : array [float] - interpolated flatfield data - """ - - self._check_interpolators(tel_id) - - ffield = self._interpolators[tel_id]["gain"](time) - return ffield - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "gain" - for the flatfield data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] - gain = input_table["gain"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = StepFunction( - time, gain, **self.interp_options - ) - - -class PedestalInterpolator(Interpolator): - """ - Interpolator for Pedestal data - """ - - telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["time", "pedestal"]) # TBD - expected_units = {"pedestal": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate pedestal or gain for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - pedestal : array [float] - interpolated pedestal values - """ - - self._check_interpolators(tel_id) - - pedestal = self._interpolators[tel_id]["pedestal"](time) - return pedestal - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "pedestal" - for the pedestal data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] - pedestal = input_table["pedestal"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = StepFunction( - time, pedestal, **self.interp_options - ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py index 930e1e7d73c..02f4c4ce306 100644 --- a/src/ctapipe/io/tests/test_interpolator.py +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -6,8 +6,6 @@ from astropy.time import Time from ctapipe.io.interpolation import ( - FlatFieldInterpolator, - PedestalInterpolator, PointingInterpolator, ) @@ -101,40 +99,13 @@ def test_bounds(): }, ) - table_pedestal = Table( - { - "time": np.arange(0.0, 10.1, 2.0), - "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) - * u.Unit(), - }, - ) - - table_flatfield = Table( - { - "time": np.arange(0.0, 10.1, 2.0), - "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) - * u.Unit(), - }, - ) - interpolator_pointing = PointingInterpolator() - interpolator_pedestal = PedestalInterpolator() - interpolator_flatfield = FlatFieldInterpolator() interpolator_pointing.add_table(1, table_pointing) - interpolator_pedestal.add_table(1, table_pedestal) - interpolator_flatfield.add_table(1, table_flatfield) - error_message = "below the interpolation range" with pytest.raises(ValueError, match=error_message): interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) - with pytest.raises(ValueError, match=error_message): - interpolator_pedestal(tel_id=1, time=-0.1) - - with pytest.raises(ValueError, match=error_message): - interpolator_flatfield(tel_id=1, time=-0.1) - with pytest.raises(ValueError, match="above the interpolation range"): interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) @@ -142,32 +113,16 @@ def test_bounds(): assert u.isclose(alt, 69 * u.deg) assert u.isclose(az, 1 * u.deg) - pedestal = interpolator_pedestal(tel_id=1, time=1.0) - assert all(pedestal == table_pedestal["pedestal"][0]) - flatfield = interpolator_flatfield(tel_id=1, time=1.0) - assert all(flatfield == table_flatfield["gain"][0]) with pytest.raises(KeyError): interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) - with pytest.raises(KeyError): - interpolator_pedestal(tel_id=2, time=1.0) - with pytest.raises(KeyError): - interpolator_flatfield(tel_id=2, time=1.0) interpolator_pointing = PointingInterpolator(bounds_error=False) - interpolator_pedestal = PedestalInterpolator(bounds_error=False) - interpolator_flatfield = FlatFieldInterpolator(bounds_error=False) interpolator_pointing.add_table(1, table_pointing) - interpolator_pedestal.add_table(1, table_pedestal) - interpolator_flatfield.add_table(1, table_flatfield) - for dt in (-0.1, 10.1) * u.s: alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) assert np.isnan(alt.value) assert np.isnan(az.value) - assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) - assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) - interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) interpolator_pointing.add_table(1, table_pointing) alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) From 92f0fb83b57d2c2d5eba531ce9b8aa5c0d67aa1d Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:50:51 +0200 Subject: [PATCH 074/221] Fixed some issues with the ChunkFunction --- src/ctapipe/calib/camera/calibrator.py | 4 +- src/ctapipe/io/interpolation.py | 62 ++++++++++++++++------- src/ctapipe/io/tests/test_interpolator.py | 9 +++- 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6c72ff5fa6a..98f58454f5b 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -9,7 +9,7 @@ import astropy.units as u import numpy as np -import Vizier +import Vizier # discuss this dependency with max etc. from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 @@ -298,8 +298,6 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # TODO: Currently not in the dependency list of ctapipe - self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py index 3b792c2107d..e0e27470c99 100644 --- a/src/ctapipe/io/interpolation.py +++ b/src/ctapipe/io/interpolation.py @@ -12,12 +12,13 @@ from .astropy_helpers import read_table -class StepFunction: +class ChunkFunction: """ - Step function Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the closest previous - point given. + Chunk Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the latest starting + valid chunk is given or the earliest available still valid chunk for any + pixels without valid data. Parameters ---------- @@ -31,7 +32,8 @@ class StepFunction: def __init__( self, - times, + start_times, + end_times, values, bounds_error=True, fill_value="extrapolate", @@ -39,12 +41,13 @@ def __init__( copy=False, ): self.values = values - self.times = times + self.start_times = start_times + self.end_times = end_times self.bounds_error = bounds_error self.fill_value = fill_value def __call__(self, point): - if point < self.times[0]: + if point < self.start_times[0]: if self.bounds_error: raise ValueError("below the interpolation range") @@ -56,9 +59,28 @@ def __call__(self, point): a[:] = np.nan return a + elif point > self.end_times[-1]: + if self.bounds_error: + raise ValueError("above the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[-1] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + else: - i = np.searchsorted(self.times, point, side="left") - return self.values[i - 1] + i = np.searchsorted( + self.start_times, point, side="left" + ) # Latest valid chunk + j = np.searchsorted( + self.end_times, point, side="left" + ) # Earliest valid chunk + return np.where( + np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] + ) # Give value for latest chunk unless its nan. If nan give earliest chunk value class Interpolator(Component, metaclass=ABCMeta): @@ -237,7 +259,7 @@ class FlatFieldInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["time", "gain"]) # TBD + required_columns = frozenset(["start_time", "end_time", "gain"]) expected_units = {"gain": u.one} def __call__(self, tel_id, time): @@ -279,12 +301,13 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] gain = input_table["gain"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = StepFunction( - time, gain, **self.interp_options + self._interpolators[tel_id]["gain"] = ChunkFunction( + start_time, end_time, gain, **self.interp_options ) @@ -294,7 +317,7 @@ class PedestalInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["time", "pedestal"]) # TBD + required_columns = frozenset(["start_time", "end_time", "pedestal"]) expected_units = {"pedestal": u.one} def __call__(self, tel_id, time): @@ -336,10 +359,11 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] pedestal = input_table["pedestal"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = StepFunction( - time, pedestal, **self.interp_options + self._interpolators[tel_id]["pedestal"] = ChunkFunction( + start_time, end_time, pedestal, **self.interp_options ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py index 930e1e7d73c..20f5657c1ae 100644 --- a/src/ctapipe/io/tests/test_interpolator.py +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -103,7 +103,8 @@ def test_bounds(): table_pedestal = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -111,7 +112,8 @@ def test_bounds(): table_flatfield = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -168,6 +170,9 @@ def test_bounds(): assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=20.0))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=20.0))) + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) interpolator_pointing.add_table(1, table_pointing) alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) From 2566a4d8c184bbc60f170176fc2c65643fc70c85 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:53:05 +0200 Subject: [PATCH 075/221] Adding the StatisticsExtractors --- src/ctapipe/calib/camera/extractor.py | 233 ++++++++++++++++++ .../calib/camera/tests/test_extractors.py | 84 +++++++ 2 files changed, 317 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..7093d057f20 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,233 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", + "SigmaClippingExtractor", +] + +from abc import abstractmethod + +import numpy as np +from astropy.stats import sigma_clipped_stats + +from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + Int, + List, +) + + +class StatisticsExtractor(TelescopeComponent): + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape + (n_images, n_channels, n_pix). + col_name : string + column name in the dl1 table + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + # in python 3.12 itertools.batched can be used + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) + return stats_list + + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + # ensure numpy array + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + # median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + ) + + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + # in python 3.12 itertools.batched can be used + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + # ensure numpy array + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + + # median of the event images + # image_median = np.ma.median(masked_images, axis=-1) + + # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + masked_images, + sigma=max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + # outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + std_outliers=image_std_outliers.filled(True), + ) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..a83c93fd1c0 --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,84 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import numpy as np +import pytest +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + + +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): + """test basic functionality of the StatisticsExtractors""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) + + +def test_check_outliers(test_sigmaclippingextractor): + """test detection ability of outliers""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 From e3113ab6e0a33abdb97fddbc7be7aab369462973 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 23 Aug 2024 18:25:44 +0200 Subject: [PATCH 076/221] implement the StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 192 ----------- src/ctapipe/io/__init__.py | 7 +- src/ctapipe/monitoring/calculator.py | 335 +++++++++++++++++++ src/ctapipe/monitoring/outlier.py | 12 +- src/ctapipe/monitoring/tests/test_outlier.py | 4 +- 5 files changed, 344 insertions(+), 206 deletions(-) create mode 100644 src/ctapipe/monitoring/calculator.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 07cd295d06c..12323587550 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -28,8 +28,6 @@ from ctapipe.io import TableLoader __all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", "CameraCalibrator", ] @@ -59,196 +57,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index 7974f1ffaaa..83a06dd5dce 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,12 +18,7 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter -from .interpolation import ( - Interpolator, - PointingInterpolator, - FlatFieldInterpolator, - PedestalInterpolator, -) +from .interpolation import Interpolator __all__ = [ "HDF5TableWriter", diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py new file mode 100644 index 00000000000..26b5dfd6377 --- /dev/null +++ b/src/ctapipe/monitoring/calculator.py @@ -0,0 +1,335 @@ +""" +Definition of the ``CalibrationCalculator`` classes, providing all steps needed to +calculate the montoring data for the camera calibration. +""" + +import pathlib +from abc import abstractmethod + +import numpy as np +from astropy.table import vstack + +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + Bool, + CaselessStrEnum, + ComponentName, + Dict, + Float, + Int, + List, + Path, + TelescopeParameter, +) +from ctapipe.io import write_table +from ctapipe.io.tableloader import TableLoader +from ctapipe.monitoring.aggregator import StatisticsAggregator +from ctapipe.monitoring.outlier import OutlierDetector + +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", +] + +PEDESTAL_GROUP = "/dl0/monitoring/telescope/pedestal" +FLATFIELD_GROUP = "/dl0/monitoring/telescope/flatfield" +TIMECALIB_GROUP = "/dl0/monitoring/telescope/time_calibration" + + +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_aggregator: str + The name of the StatisticsAggregator subclass to be used to aggregate the statistics + """ + + stats_aggregator_type = TelescopeParameter( + trait=ComponentName( + StatisticsAggregator, default_value="SigmaClippingAggregator" + ), + default_value="SigmaClippingAggregator", + help="Name of the StatisticsAggregator subclass to be used.", + ).tag(config=True) + + outlier_detector_type = List( + trait=Dict, + default_value=None, + allow_none=True, + help=( + "List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector." + ), + ).tag(config=True) + + calibration_type = CaselessStrEnum( + ["pedestal", "flatfield", "time_calibration"], + allow_none=False, + help="Set type of calibration which is needed to properly store the monitoring data", + ).tag(config=True) + + output_path = Path( + help="output filename", default_value=pathlib.Path("monitoring.camcalib.h5") + ).tag(config=True) + + overwrite = Bool(help="overwrite output file if it exists").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_aggregator=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_aggregator: ctapipe.monitoring.aggregator.StatisticsAggregator + The StatisticsAggregator to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.group = { + "pedestal": PEDESTAL_GROUP, + "flatfield": FLATFIELD_GROUP, + "time_calibration": TIMECALIB_GROUP, + } + + # Initialize the instances of StatisticsAggregator + self.stats_aggregator = {} + if stats_aggregator is None: + for _, _, name in self.stats_aggregator_type: + self.stats_aggregator[name] = StatisticsAggregator.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_aggregator.__class__.__name__ + self.stats_aggregator_type = [("type", "*", name)] + self.stats_aggregator[name] = stats_aggregator + + # Initialize the instances of OutlierDetector + self.outlier_detectors = {} + if self.outlier_detector_type is not None: + for outlier_detector in self.outlier_detector_type: + self.outlier_detectors[outlier_detector["apply_to"]] = ( + OutlierDetector.from_name( + name=outlier_detector["name"], + validity_range=outlier_detector["validity_range"], + subarray=self.subarray, + parent=self, + ) + ) + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class StatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + chunk_shift = Int( + default_value=None, + allow_none=True, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + two_pass = Bool(default_value=False, help="overwrite output file if it exists").tag( + config=True + ) + + faulty_pixels_threshold = Float( + default_value=0.1, + allow_none=True, + help=( + "Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk" + ), + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + + # Check if the chunk_shift is set for two pass mode + if self.two_pass and self.chunk_shift is None: + raise ValueError("chunk_shift must be set for two pass mode") + + # Get the aggregator + aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] + # Pass through the whole provided dl1 data + if self.two_pass: + self.aggregated_stats = aggregator( + table=dl1_table[tel_id], col_name=col_name, chunk_shift=None + ) + else: + self.aggregated_stats = aggregator( + table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift + ) + # Detect faulty pixels with mutiple instances of OutlierDetector + outlier_mask = np.zeros_like(self.aggregated_stats[0]["mean"], dtype=bool) + for aggregated_val, outlier_detector in self.outlier_detectors.items(): + outlier_mask = np.logical_or( + outlier_mask, + outlier_detector(self.aggregated_stats[aggregated_val]), + ) + # Add the outlier mask to the aggregated statistics + self.aggregated_stats["outlier_mask"] = outlier_mask + + if self.two_pass: + # Check if the camera has two gain channels + if outlier_mask.shape[1] == 2: + # Combine the outlier mask of both gain channels + outlier_mask = np.logical_or( + outlier_mask[:, 0, :], + outlier_mask[:, 1, :], + ) + # Calculate the fraction of faulty pixels over the camera + faulty_pixels_percentage = ( + np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] + ) + + # Check for faulty chunks if the threshold is exceeded + faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold + if np.any(faulty_chunks): + faulty_chunks_indices = np.where(faulty_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunks ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={self.aggregated_stats['time_start'][index]}; time_end={self.aggregated_stats['time_end'][index]}" + ) + + # Slice the dl1 table according to the previously caluclated start and end. + slice_start, slice_end = self._get_slice_range( + chunk_index=index, + faulty_previous_chunk=(index-1 in faulty_chunks_indices), + dl1_table_length=len(dl1_table[tel_id]) - 1, + ) + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] + + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + aggregated_stats_secondpass = aggregator( + table=dl1_table_sliced, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) + + # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass + outlier_mask = np.zeros_like(aggregated_stats_secondpass[0]["mean"], dtype=bool) + for aggregated_val, outlier_detector in self.outlier_detectors.items(): + outlier_mask = np.logical_or( + outlier_mask, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), + ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask + + # Stack the aggregated statistics of the second pass to the first pass + self.aggregated_stats = vstack([self.aggregated_stats, aggregated_stats_secondpass]) + # Sort the aggregated statistics based on the starting time + self.aggregated_stats.sort(["time_start"]) + else: + self.log.info( + "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + ) + + # Write the aggregated statistics and their outlier mask to the output file + write_table( + self.aggregated_stats, + self.output_path, + f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", + overwrite=self.overwrite, + ) + + def _get_slice_range( + self, + chunk_index, + faulty_previous_chunk, + dl1_table_length, + ) -> (int, int): + """ + Calculate the start and end indices for slicing the DL1 table to be used for the second pass. + + Parameters + ---------- + chunk_index : int + The index of the current faulty chunk being processed. + faulty_previous_chunk : bool + A flag indicating if the previous chunk was faulty. + dl1_table_length : int + The total length of the DL1 table. + + Returns + ------- + tuple + A tuple containing the start and end indices for slicing the DL1 table. + """ + + # Set the start of the slice to the first element of the dl1 table + slice_start = 0 + if chunk_index > 0: + # Get the start of the previous chunk + if faulty_previous_chunk: + slice_start = np.sum(self.aggregated_stats["n_events"][:chunk_index]) + else: + slice_start = np.sum( + self.aggregated_stats["n_events"][: chunk_index - 1] + ) + + # Set the end of the slice to the last element of the dl1 table + slice_end = dl1_table_length + if chunk_index < len(self.aggregated_stats) - 1: + # Get the stop of the next chunk + slice_end = np.sum(self.aggregated_stats["n_events"][: chunk_index + 2]) + + # Shift the start and end of the slice by the chunk_shift + slice_start += self.chunk_shift + slice_end -= self.chunk_shift - 1 + + return int(slice_start), int(slice_end) diff --git a/src/ctapipe/monitoring/outlier.py b/src/ctapipe/monitoring/outlier.py index 71bcfd36c86..93b67bfd418 100644 --- a/src/ctapipe/monitoring/outlier.py +++ b/src/ctapipe/monitoring/outlier.py @@ -82,7 +82,7 @@ class MedianOutlierDetector(OutlierDetector): the configurable factors and the camera median of the statistic values. """ - median_range_factors = List( + validity_range = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -99,8 +99,8 @@ def __call__(self, column): # Detect outliers based on the deviation of the median distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.median_range_factors[0] * camera_median[:, :, np.newaxis], - deviation > self.median_range_factors[1] * camera_median[:, :, np.newaxis], + deviation < self.validity_range[0] * camera_median[:, :, np.newaxis], + deviation > self.validity_range[1] * camera_median[:, :, np.newaxis], ) return outliers @@ -113,7 +113,7 @@ class StdOutlierDetector(OutlierDetector): the configurable factors and the camera standard deviation of the statistic values. """ - std_range_factors = List( + validity_range = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -132,7 +132,7 @@ def __call__(self, column): # Detect outliers based on the deviation of the standard deviation distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.std_range_factors[0] * camera_std[:, :, np.newaxis], - deviation > self.std_range_factors[1] * camera_std[:, :, np.newaxis], + deviation < self.validity_range[0] * camera_std[:, :, np.newaxis], + deviation > self.validity_range[1] * camera_std[:, :, np.newaxis], ) return outliers diff --git a/src/ctapipe/monitoring/tests/test_outlier.py b/src/ctapipe/monitoring/tests/test_outlier.py index da7d7619b33..61f1d8cb91d 100644 --- a/src/ctapipe/monitoring/tests/test_outlier.py +++ b/src/ctapipe/monitoring/tests/test_outlier.py @@ -56,7 +56,7 @@ def test_median_detection(example_subarray): # In this test, the interval [-0.9, 8] corresponds to multiplication factors # typical used for the median values of charge images of flat-field events detector = MedianOutlierDetector( - subarray=example_subarray, median_range_factors=[-0.9, 8.0] + subarray=example_subarray, validity_range=[-0.9, 8.0] ) # Detect outliers outliers = detector(table["median"]) @@ -89,7 +89,7 @@ def test_std_detection(example_subarray): # typical used for the std values of charge images of flat-field events # and median (and std) values of charge images of pedestal events detector = StdOutlierDetector( - subarray=example_subarray, std_range_factors=[-15.0, 15.0] + subarray=example_subarray, validity_range=[-15.0, 15.0] ) ff_outliers = detector(ff_table["std"]) ped_outliers = detector(ped_table["median"]) From b9dc929d3992c3fa98ca2aab6653789a2fb6463e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sat, 24 Aug 2024 11:19:38 +0200 Subject: [PATCH 077/221] removed the helper function to get the start and end slices Since agregation is chunk has always the same n_events, we can simplify the retrieving of the start and end slices. Therefore we do not need a helper function anymore --- src/ctapipe/monitoring/calculator.py | 132 +++++++++++---------------- 1 file changed, 53 insertions(+), 79 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 26b5dfd6377..56de565c567 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -159,18 +159,19 @@ class StatisticsCalculator(CalibrationCalculator): chunk_shift = Int( default_value=None, allow_none=True, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + help="Number of samples to shift the aggregation chunk for the calculation of the statistical values", ).tag(config=True) - two_pass = Bool(default_value=False, help="overwrite output file if it exists").tag( - config=True - ) + second_pass = Bool( + default_value=False, help="overwrite output file if it exists" + ).tag(config=True) faulty_pixels_threshold = Float( - default_value=0.1, + default_value=10.0, allow_none=True, help=( - "Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk" + "Threshold in percentage of faulty pixels over the camera " + "to conduct second pass with a refined shift of the chunk." ), ).tag(config=True) @@ -197,31 +198,33 @@ def __call__( ) # Check if the chunk_shift is set for two pass mode - if self.two_pass and self.chunk_shift is None: - raise ValueError("chunk_shift must be set for two pass mode") + if self.second_pass and self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is selected" + ) # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 data - if self.two_pass: - self.aggregated_stats = aggregator( + if self.second_pass: + aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=None ) else: - self.aggregated_stats = aggregator( + aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift ) # Detect faulty pixels with mutiple instances of OutlierDetector - outlier_mask = np.zeros_like(self.aggregated_stats[0]["mean"], dtype=bool) + outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( outlier_mask, - outlier_detector(self.aggregated_stats[aggregated_val]), + outlier_detector(aggregated_stats[aggregated_val]), ) # Add the outlier mask to the aggregated statistics - self.aggregated_stats["outlier_mask"] = outlier_mask + aggregated_stats["outlier_mask"] = outlier_mask - if self.two_pass: + if self.second_pass: # Check if the camera has two gain channels if outlier_mask.shape[1] == 2: # Combine the outlier mask of both gain channels @@ -232,24 +235,33 @@ def __call__( # Calculate the fraction of faulty pixels over the camera faulty_pixels_percentage = ( np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] - ) + ) * 100.0 # Check for faulty chunks if the threshold is exceeded faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold if np.any(faulty_chunks): + chunk_size = aggregated_stats["n_events"][0] faulty_chunks_indices = np.where(faulty_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.warning( - f"Faulty chunks ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={self.aggregated_stats['time_start'][index]}; time_end={self.aggregated_stats['time_end'][index]}" + f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - - # Slice the dl1 table according to the previously caluclated start and end. - slice_start, slice_end = self._get_slice_range( - chunk_index=index, - faulty_previous_chunk=(index-1 in faulty_chunks_indices), - dl1_table_length=len(dl1_table[tel_id]) - 1, + # Calculate the start of the slice based + slice_start = ( + chunk_size * index + if index - 1 in faulty_chunks_indices + else chunk_size * (index - 1) ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min( + len(dl1_table[tel_id]) - 1, chunk_size * (index + 2) + ) - (self.chunk_shift - 1) + # Slice the dl1 table according to the previously caluclated start and end. dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift @@ -261,19 +273,28 @@ def __call__( ) # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass - outlier_mask = np.zeros_like(aggregated_stats_secondpass[0]["mean"], dtype=bool) - for aggregated_val, outlier_detector in self.outlier_detectors.items(): - outlier_mask = np.logical_or( - outlier_mask, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass[0]["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector( + aggregated_stats_secondpass[aggregated_val] + ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass # Stack the aggregated statistics of the second pass to the first pass - self.aggregated_stats = vstack([self.aggregated_stats, aggregated_stats_secondpass]) + aggregated_stats = vstack( + [aggregated_stats, aggregated_stats_secondpass] + ) # Sort the aggregated statistics based on the starting time - self.aggregated_stats.sort(["time_start"]) + aggregated_stats.sort(["time_start"]) else: self.log.info( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." @@ -281,55 +302,8 @@ def __call__( # Write the aggregated statistics and their outlier mask to the output file write_table( - self.aggregated_stats, + aggregated_stats, self.output_path, f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", overwrite=self.overwrite, ) - - def _get_slice_range( - self, - chunk_index, - faulty_previous_chunk, - dl1_table_length, - ) -> (int, int): - """ - Calculate the start and end indices for slicing the DL1 table to be used for the second pass. - - Parameters - ---------- - chunk_index : int - The index of the current faulty chunk being processed. - faulty_previous_chunk : bool - A flag indicating if the previous chunk was faulty. - dl1_table_length : int - The total length of the DL1 table. - - Returns - ------- - tuple - A tuple containing the start and end indices for slicing the DL1 table. - """ - - # Set the start of the slice to the first element of the dl1 table - slice_start = 0 - if chunk_index > 0: - # Get the start of the previous chunk - if faulty_previous_chunk: - slice_start = np.sum(self.aggregated_stats["n_events"][:chunk_index]) - else: - slice_start = np.sum( - self.aggregated_stats["n_events"][: chunk_index - 1] - ) - - # Set the end of the slice to the last element of the dl1 table - slice_end = dl1_table_length - if chunk_index < len(self.aggregated_stats) - 1: - # Get the stop of the next chunk - slice_end = np.sum(self.aggregated_stats["n_events"][: chunk_index + 2]) - - # Shift the start and end of the slice by the chunk_shift - slice_start += self.chunk_shift - slice_end -= self.chunk_shift - 1 - - return int(slice_start), int(slice_end) From ef920da04b52e2f460dc65f6cb042a398bace465 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:01:41 +0200 Subject: [PATCH 078/221] polish docstrings --- CODEOWNERS | 2 + src/ctapipe/monitoring/calculator.py | 70 +++++++++++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 06c29bc6c78..70c53117a07 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,6 +5,8 @@ ctapipe/calib/camera @watsonjj ctapipe/image/extractor.py @watsonjj @HealthyPear +ctapipe/monitoring @TjarkMiener + ctapipe/reco/HillasReconstructor.py @HealthyPear ctapipe/reco/tests/test_HillasReconstructor.py @HealthyPear diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 56de565c567..efccfb11368 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -38,12 +38,25 @@ class CalibrationCalculator(TelescopeComponent): """ - Base component for various calibration calculators + Base component for calibration calculators. + + This class provides the foundational methods and attributes for + calculating camera-related monitoring data. It is designed + to be extended by specific calibration calculators that implement + the required methods for different types of calibration. Attributes ---------- - stats_aggregator: str - The name of the StatisticsAggregator subclass to be used to aggregate the statistics + stats_aggregator_type : ctapipe.core.traits.TelescopeParameter + The type of StatisticsAggregator to be used for aggregating statistics. + outlier_detector_list : list of dict + List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. + calibration_type : ctapipe.core.traits.CaselessStrEnum + The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. + output_path : ctapipe.core.traits.Path + The output filename where the calibration data will be stored. + overwrite : ctapipe.core.traits.Bool + Whether to overwrite the output file if it exists. """ stats_aggregator_type = TelescopeParameter( @@ -54,12 +67,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsAggregator subclass to be used.", ).tag(config=True) - outlier_detector_type = List( + outlier_detector_list = List( trait=Dict, default_value=None, allow_none=True, help=( - "List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector." + "List of dicts containing the name of the OutlierDetector subclass to be used, " + "the aggregated value to which the detector should be applied, " + "and the validity range of the detector." ), ).tag(config=True) @@ -70,10 +85,10 @@ class CalibrationCalculator(TelescopeComponent): ).tag(config=True) output_path = Path( - help="output filename", default_value=pathlib.Path("monitoring.camcalib.h5") + help="Output filename", default_value=pathlib.Path("monitoring.camcalib.h5") ).tag(config=True) - overwrite = Bool(help="overwrite output file if it exists").tag(config=True) + overwrite = Bool(help="Overwrite output file if it exists").tag(config=True) def __init__( self, @@ -98,7 +113,7 @@ def __init__( Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` stats_aggregator: ctapipe.monitoring.aggregator.StatisticsAggregator - The StatisticsAggregator to use. If None, the default via the + The ``StatisticsAggregator`` to use. If None, the default via the configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @@ -124,8 +139,8 @@ def __init__( # Initialize the instances of OutlierDetector self.outlier_detectors = {} - if self.outlier_detector_type is not None: - for outlier_detector in self.outlier_detector_type: + if self.outlier_detector_list is not None: + for outlier_detector in self.outlier_detector_list: self.outlier_detectors[outlier_detector["apply_to"]] = ( OutlierDetector.from_name( name=outlier_detector["name"], @@ -136,7 +151,7 @@ def __init__( ) @abstractmethod - def __call__(self, input_url, tel_id): + def __call__(self, input_url, tel_id, col_name): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -154,16 +169,35 @@ def __call__(self, input_url, tel_id): class StatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. + + This class inherits from CalibrationCalculator and is responsible for + calculating various statistics from calibration events, such as pedestal + and flat-field data. It reads the data, aggregates statistics, detects + outliers, handles faulty data chunks, and stores the monitoring data. + The default option is to conduct only one pass over the data with non-overlapping + chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. + Two passes over the data, set via the ``second_pass``-flag, can be conducted + with a refined shift of the chunk in regions of trouble with a high percentage + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( default_value=None, allow_none=True, - help="Number of samples to shift the aggregation chunk for the calculation of the statistical values", + help=( + "Number of samples to shift the aggregation chunk for the " + "calculation of the statistical values. If second_pass is set, " + "the first pass is conducted without overlapping chunks (chunk_shift=None) " + "and the second pass with a refined shift of the chunk in regions of trouble." + ), ).tag(config=True) second_pass = Bool( - default_value=False, help="overwrite output file if it exists" + default_value=False, + help=( + "Set whether to conduct a second pass over the data " + "with a refined shift of the chunk in regions of trouble." + ), ).tag(config=True) faulty_pixels_threshold = Float( @@ -171,7 +205,8 @@ class StatisticsCalculator(CalibrationCalculator): allow_none=True, help=( "Threshold in percentage of faulty pixels over the camera " - "to conduct second pass with a refined shift of the chunk." + "to conduct second pass with a refined shift of the chunk " + "in regions of trouble." ), ).tag(config=True) @@ -182,7 +217,7 @@ def __call__( col_name="image", ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( telescopes=tel_id, @@ -197,7 +232,7 @@ def __call__( pointing=False, ) - # Check if the chunk_shift is set for two pass mode + # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: raise ValueError( "chunk_shift must be set if second pass over the data is selected" @@ -214,7 +249,7 @@ def __call__( aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift ) - # Detect faulty pixels with mutiple instances of OutlierDetector + # Detect faulty pixels with mutiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( @@ -224,6 +259,7 @@ def __call__( # Add the outlier mask to the aggregated statistics aggregated_stats["outlier_mask"] = outlier_mask + # Conduct a second pass over the data if self.second_pass: # Check if the camera has two gain channels if outlier_mask.shape[1] == 2: From 9bfc16aaa5f57499fad1578a28421b618e1164d1 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:13:27 +0200 Subject: [PATCH 079/221] further polishing of docstrings --- src/ctapipe/monitoring/calculator.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index efccfb11368..9d948bde388 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -54,7 +54,7 @@ class CalibrationCalculator(TelescopeComponent): calibration_type : ctapipe.core.traits.CaselessStrEnum The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. output_path : ctapipe.core.traits.Path - The output filename where the calibration data will be stored. + The output filename where the monitoring data will be stored. overwrite : ctapipe.core.traits.Bool Whether to overwrite the output file if it exists. """ @@ -153,16 +153,21 @@ def __init__( @abstractmethod def __call__(self, input_url, tel_id, col_name): """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events + Calculate the monitoring data for a given set of events. + + This method should be implemented by subclasses to perform the specific + calibration calculations required for different types of calibration. Parameters ---------- input_url : str - URL where the events are stored from which the calibration coefficients + URL where the events are stored from which the monitoring data are to be calculated tel_id : int - The telescope id + The telescope ID for which the calibration is being performed. + col_name : str + The name of the column in the data from which the statistics + will be aggregated. """ @@ -283,7 +288,7 @@ def __call__( self.log.warning( f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - # Calculate the start of the slice based + # Calculate the start of the slice based weather the previous chunk was faulty or not slice_start = ( chunk_size * index if index - 1 in faulty_chunks_indices From aee5c10bc4638966bc26458013d598dedc8319e4 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:17:28 +0200 Subject: [PATCH 080/221] fix typo --- src/ctapipe/monitoring/calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 9d948bde388..b6fdd3e0417 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -288,7 +288,7 @@ def __call__( self.log.warning( f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - # Calculate the start of the slice based weather the previous chunk was faulty or not + # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( chunk_size * index if index - 1 in faulty_chunks_indices From 1484ecaad3c407ef6c7313270a0408e7452abc95 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:20:38 +0200 Subject: [PATCH 081/221] move check of config settings before loading of data --- src/ctapipe/monitoring/calculator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index b6fdd3e0417..09bb1a0ab01 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -222,6 +222,12 @@ def __call__( col_name="image", ): + # Check if the chunk_shift is set for second pass mode + if self.second_pass and self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is selected" + ) + # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( @@ -237,12 +243,6 @@ def __call__( pointing=False, ) - # Check if the chunk_shift is set for second pass mode - if self.second_pass and self.chunk_shift is None: - raise ValueError( - "chunk_shift must be set if second pass over the data is selected" - ) - # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 data From 601cbb42cf676546f92dcae8da04d6ca5bc63849 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 16:04:31 +0200 Subject: [PATCH 082/221] moved Interpolator outside This branch should only host the devs for the stats calculator --- src/ctapipe/calib/camera/calibrator.py | 11 +- src/ctapipe/calib/camera/extractor.py | 233 ------------------ .../calib/camera/tests/test_extractors.py | 84 ------- src/ctapipe/io/__init__.py | 5 +- src/ctapipe/io/interpolation.py | 182 -------------- src/ctapipe/monitoring/calculator.py | 2 +- 6 files changed, 3 insertions(+), 514 deletions(-) delete mode 100644 src/ctapipe/calib/camera/extractor.py delete mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py delete mode 100644 src/ctapipe/io/interpolation.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 12323587550..853ba3f7da8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -3,33 +3,24 @@ calibration and image extraction, as well as supporting algorithms. """ -import pickle -from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Float, - Int, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import TableLoader -__all__ = [ - "CameraCalibrator", -] +__all__ = ["CameraCalibrator"] @cache diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py deleted file mode 100644 index 7093d057f20..00000000000 --- a/src/ctapipe/calib/camera/extractor.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -Extraction algorithms to compute the statistics from a sequence of images -""" - -__all__ = [ - "StatisticsExtractor", - "PlainExtractor", - "SigmaClippingExtractor", -] - -from abc import abstractmethod - -import numpy as np -from astropy.stats import sigma_clipped_stats - -from ctapipe.containers import StatisticsContainer -from ctapipe.core import TelescopeComponent -from ctapipe.core.traits import ( - Int, - List, -) - - -class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) - image_median_cut_outliers = List( - [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", - ).tag(config=True) - image_std_cut_outliers = List( - [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", - ).tag(config=True) - - def __init__(self, subarray, config=None, parent=None, **kwargs): - """ - Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). - - Parameters - ---------- - kwargs - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - @abstractmethod - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - """ - Call the relevant functions to extract the statistics - for the particular extractor. - - Parameters - ---------- - dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape - (n_images, n_channels, n_pix). - col_name : string - column name in the dl1 table - - Returns - ------- - List StatisticsContainer: - List of extracted statistics and validity ranges - """ - - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _plain_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median over the sample per pixel - pixel_median = np.ma.median(masked_images, axis=0) - - # mean over the sample per pixel - pixel_mean = np.ma.mean(masked_images, axis=0) - - # std over the sample per pixel - pixel_std = np.ma.std(masked_images, axis=0) - - # median of the median over the camera - # median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # outliers from median - image_median_outliers = np.logical_or( - pixel_median < self.image_median_cut_outliers[0], - pixel_median > self.image_median_cut_outliers[1], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - ) - - -class SigmaClippingExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using astropy's sigma clipping functions - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median of the event images - # image_median = np.ma.median(masked_images, axis=-1) - - # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - # mask pixels without defined statistical values - pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) - pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) - pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - - # only warn for values discard in the sigma clipping, not those from before - # outliers = unused_values & (~masked_images.mask) - - # add outliers identified by sigma clipping for following operations - masked_images.mask |= unused_values - - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # median of the std over the camera - median_of_pixel_std = np.ma.median(pixel_std, axis=1) - - # std of the std over camera - std_of_pixel_std = np.ma.std(pixel_std, axis=1) - - # outliers from median - image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = np.logical_or( - image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], - image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], - ) - - # outliers from standard deviation - deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = np.logical_or( - deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - std_outliers=image_std_outliers.filled(True), - ) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py deleted file mode 100644 index a83c93fd1c0..00000000000 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Tests for StatisticsExtractor and related functions -""" - -import numpy as np -import pytest -from astropy.table import QTable - -from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor - - -@pytest.fixture(name="test_plainextractor") -def fixture_test_plainextractor(example_subarray): - """test the PlainExtractor""" - return PlainExtractor(subarray=example_subarray, chunk_size=2500) - - -@pytest.fixture(name="test_sigmaclippingextractor") -def fixture_test_sigmaclippingextractor(example_subarray): - """test the SigmaClippingExtractor""" - return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) - - -def test_extractors(test_plainextractor, test_sigmaclippingextractor): - """test basic functionality of the StatisticsExtractors""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - - plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) - - -def test_check_outliers(test_sigmaclippingextractor): - """test detection ability of outliers""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_data[:, 0, 120] = 120.0 - flatfield_dl1_data[:, 1, 67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] - assert sigmaclipping_stats_list[0].median_outliers[1][67] - assert sigmaclipping_stats_list[1].median_outliers[0][120] - assert sigmaclipping_stats_list[1].median_outliers[1][67] - - -def test_check_chunk_shift(test_sigmaclippingextractor): - """test the chunk shift option and the boundary case for the last chunk""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, chunk_shift=2000 - ) - - # check if three chunks are used for the extraction - assert len(sigmaclipping_stats_list) == 3 diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index 83a06dd5dce..229f212b766 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,7 +18,7 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter -from .interpolation import Interpolator +from .pointing import PointingInterpolator __all__ = [ "HDF5TableWriter", @@ -37,8 +37,5 @@ "DataWriter", "DATA_MODEL_VERSION", "get_hdf5_datalevels", - "Interpolator", "PointingInterpolator", - "PedestalInterpolator", - "FlatFieldInterpolator", ] diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py deleted file mode 100644 index 82de9f0bd19..00000000000 --- a/src/ctapipe/io/interpolation.py +++ /dev/null @@ -1,182 +0,0 @@ -from abc import ABCMeta, abstractmethod -from typing import Any - -import astropy.units as u -import numpy as np -import tables -from astropy.time import Time -from scipy.interpolate import interp1d - -from ctapipe.core import Component, traits - -from .astropy_helpers import read_table - - -class Interpolator(Component, metaclass=ABCMeta): - """ - Interpolator parent class. - - Parameters - ---------- - h5file : None | tables.File - A open hdf5 file with read access. - """ - - bounds_error = traits.Bool( - default_value=True, - help="If true, raises an exception when trying to extrapolate out of the given table", - ).tag(config=True) - - extrapolate = traits.Bool( - help="If bounds_error is False, this flag will specify whether values outside" - "the available values are filled with nan (False) or extrapolated (True).", - default_value=False, - ).tag(config=True) - - telescope_data_group = None - required_columns = set() - expected_units = {} - - def __init__(self, h5file=None, **kwargs): - super().__init__(**kwargs) - - if h5file is not None and not isinstance(h5file, tables.File): - raise TypeError("h5file must be a tables.File") - self.h5file = h5file - - self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) - if self.bounds_error: - self.interp_options["bounds_error"] = True - elif self.extrapolate: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = "extrapolate" - else: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = np.nan - - self._interpolators = {} - - @abstractmethod - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - This method reads input tables and creates instances of the needed interpolators - to be added to _interpolators. The first index of _interpolators needs to be - tel_id, the second needs to be the name of the parameter that is to be interpolated - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are always ``time`` as ``Time`` column and - other columns for the data that is to be interpolated - """ - - pass - - def _check_tables(self, input_table): - missing = self.required_columns - set(input_table.colnames) - if len(missing) > 0: - raise ValueError(f"Table is missing required column(s): {missing}") - for col in self.expected_units: - unit = input_table[col].unit - if unit is None: - if self.expected_units[col] is not None: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - elif not self.expected_units[col].is_equivalent(unit): - if self.expected_units[col] is None: - raise ValueError(f"{col} must have units compatible with 'None'") - else: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - - def _check_interpolators(self, tel_id): - if tel_id not in self._interpolators: - if self.h5file is not None: - self._read_parameter_table(tel_id) # might need to be removed - else: - raise KeyError(f"No table available for tel_id {tel_id}") - - def _read_parameter_table(self, tel_id): - input_table = read_table( - self.h5file, - f"{self.telescope_data_group}/tel_{tel_id:03d}", - ) - self.add_table(tel_id, input_table) - - -class PointingInterpolator(Interpolator): - """ - Interpolator for pointing and pointing correction data - """ - - telescope_data_group = "/dl0/monitoring/telescope/pointing" - required_columns = frozenset(["time", "azimuth", "altitude"]) - expected_units = {"azimuth": u.rad, "altitude": u.rad} - - def __call__(self, tel_id, time): - """ - Interpolate alt/az for given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the pointing - - Returns - ------- - altitude : astropy.units.Quantity[deg] - interpolated altitude angle - azimuth : astropy.units.Quantity[deg] - interpolated azimuth angle - """ - - self._check_interpolators(tel_id) - - mjd = time.tai.mjd - az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) - alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) - return alt, az - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` - as quantity columns for pointing and pointing correction data. - """ - - self._check_tables(input_table) - - if not isinstance(input_table["time"], Time): - raise TypeError("'time' column of pointing table must be astropy.time.Time") - - input_table = input_table.copy() - input_table.sort("time") - - az = input_table["azimuth"].quantity.to_value(u.rad) - # prepare azimuth for interpolation by "unwrapping": i.e. turning - # [359, 1] into [359, 361]. This assumes that if we get values like - # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees - # the other way around. This should be true for all telescopes given - # the sampling speed of pointing values and their maximum movement speed. - # No telescope can turn more than 180° in 2 seconds. - az = np.unwrap(az) - alt = input_table["altitude"].quantity.to_value(u.rad) - mjd = input_table["time"].tai.mjd - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) - self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 09bb1a0ab01..dc56259425c 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -73,7 +73,7 @@ class CalibrationCalculator(TelescopeComponent): allow_none=True, help=( "List of dicts containing the name of the OutlierDetector subclass to be used, " - "the aggregated value to which the detector should be applied, " + "the aggregated statistic value to which the detector should be applied, " "and the validity range of the detector." ), ).tag(config=True) From 03471536ae762af039214042c829691b3c9b5a44 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 16:07:55 +0200 Subject: [PATCH 083/221] removed Interpolator artifacts --- src/ctapipe/io/__init__.py | 2 - src/ctapipe/io/tests/test_interpolator.py | 134 ---------------------- 2 files changed, 136 deletions(-) delete mode 100644 src/ctapipe/io/tests/test_interpolator.py diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index 229f212b766..afe96f39430 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,7 +18,6 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter -from .pointing import PointingInterpolator __all__ = [ "HDF5TableWriter", @@ -37,5 +36,4 @@ "DataWriter", "DATA_MODEL_VERSION", "get_hdf5_datalevels", - "PointingInterpolator", ] diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py deleted file mode 100644 index 02f4c4ce306..00000000000 --- a/src/ctapipe/io/tests/test_interpolator.py +++ /dev/null @@ -1,134 +0,0 @@ -import astropy.units as u -import numpy as np -import pytest -import tables -from astropy.table import Table -from astropy.time import Time - -from ctapipe.io.interpolation import ( - PointingInterpolator, -) - -t0 = Time("2022-01-01T00:00:00") - - -def test_azimuth_switchover(): - """Test pointing interpolation""" - - table = Table( - { - "time": t0 + [0, 1, 2] * u.s, - "azimuth": [359, 1, 3] * u.deg, - "altitude": [60, 61, 62] * u.deg, - }, - ) - - interpolator = PointingInterpolator() - interpolator.add_table(1, table) - - alt, az = interpolator(tel_id=1, time=t0 + 0.5 * u.s) - assert u.isclose(az, 360 * u.deg) - assert u.isclose(alt, 60.5 * u.deg) - - -def test_invalid_input(): - """Test invalid pointing tables raise nice errors""" - - wrong_time = Table( - { - "time": [1, 2, 3] * u.s, - "azimuth": [1, 2, 3] * u.deg, - "altitude": [1, 2, 3] * u.deg, - } - ) - - interpolator = PointingInterpolator() - with pytest.raises(TypeError, match="astropy.time.Time"): - interpolator.add_table(1, wrong_time) - - wrong_unit = Table( - { - "time": Time(1.7e9 + np.arange(3), format="unix"), - "azimuth": [1, 2, 3] * u.m, - "altitude": [1, 2, 3] * u.deg, - } - ) - with pytest.raises(ValueError, match="compatible with 'rad'"): - interpolator.add_table(1, wrong_unit) - - wrong_unit = Table( - { - "time": Time(1.7e9 + np.arange(3), format="unix"), - "azimuth": [1, 2, 3] * u.deg, - "altitude": [1, 2, 3], - } - ) - with pytest.raises(ValueError, match="compatible with 'rad'"): - interpolator.add_table(1, wrong_unit) - - -def test_hdf5(tmp_path): - """Test writing interpolated data to file""" - from ctapipe.io import write_table - - table = Table( - { - "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, - "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, - "altitude": np.linspace(70.0, 60.0, 6) * u.deg, - }, - ) - - path = tmp_path / "pointing.h5" - write_table(table, path, "/dl0/monitoring/telescope/pointing/tel_001") - with tables.open_file(path) as h5file: - interpolator = PointingInterpolator(h5file) - alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s) - assert u.isclose(alt, 69 * u.deg) - assert u.isclose(az, 1 * u.deg) - - -def test_bounds(): - """Test invalid pointing tables raise nice errors""" - - table_pointing = Table( - { - "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, - "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, - "altitude": np.linspace(70.0, 60.0, 6) * u.deg, - }, - ) - - interpolator_pointing = PointingInterpolator() - interpolator_pointing.add_table(1, table_pointing) - error_message = "below the interpolation range" - - with pytest.raises(ValueError, match=error_message): - interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) - - with pytest.raises(ValueError, match="above the interpolation range"): - interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) - - alt, az = interpolator_pointing(tel_id=1, time=t0 + 1 * u.s) - assert u.isclose(alt, 69 * u.deg) - assert u.isclose(az, 1 * u.deg) - - with pytest.raises(KeyError): - interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) - - interpolator_pointing = PointingInterpolator(bounds_error=False) - interpolator_pointing.add_table(1, table_pointing) - for dt in (-0.1, 10.1) * u.s: - alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) - assert np.isnan(alt.value) - assert np.isnan(az.value) - - interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) - interpolator_pointing.add_table(1, table_pointing) - alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) - assert u.isclose(alt, 71 * u.deg) - assert u.isclose(az, -1 * u.deg) - - alt, az = interpolator_pointing(tel_id=1, time=t0 + 11 * u.s) - assert u.isclose(alt, 59 * u.deg) - assert u.isclose(az, 11 * u.deg) From b2208d5b9430ecbf0bd0a1a01f7ee45b25cd9ecd Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 18:34:04 +0200 Subject: [PATCH 084/221] removed reading part with TableLoader reading should be done in the tool outside of this component --- src/ctapipe/monitoring/calculator.py | 67 +++++++++++++--------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index dc56259425c..f93a88e0b78 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -22,7 +22,6 @@ TelescopeParameter, ) from ctapipe.io import write_table -from ctapipe.io.tableloader import TableLoader from ctapipe.monitoring.aggregator import StatisticsAggregator from ctapipe.monitoring.outlier import OutlierDetector @@ -151,7 +150,7 @@ def __init__( ) @abstractmethod - def __call__(self, input_url, tel_id, col_name): + def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): """ Calculate the monitoring data for a given set of events. @@ -160,14 +159,15 @@ def __call__(self, input_url, tel_id, col_name): Parameters ---------- - input_url : str - URL where the events are stored from which the monitoring data - are to be calculated + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix) + and timestamps of shape (n_images, ) + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing tel_id : int - The telescope ID for which the calibration is being performed. + Telescope ID for which the calibration is being performed col_name : str - The name of the column in the data from which the statistics - will be aggregated. + Column name in the table from which the statistics will be aggregated """ @@ -177,13 +177,13 @@ class StatisticsCalculator(CalibrationCalculator): This class inherits from CalibrationCalculator and is responsible for calculating various statistics from calibration events, such as pedestal - and flat-field data. It reads the data, aggregates statistics, detects - outliers, handles faulty data chunks, and stores the monitoring data. + and flat-field data. It aggregates statistics, detects outliers, + handles faulty data chunks, and stores the monitoring data. The default option is to conduct only one pass over the data with non-overlapping chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. Two passes over the data, set via the ``second_pass``-flag, can be conducted with a refined shift of the chunk in regions of trouble with a high percentage - of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( @@ -217,7 +217,8 @@ class StatisticsCalculator(CalibrationCalculator): def __call__( self, - input_url, + table, + masked_pixels_of_sample, tel_id, col_name="image", ): @@ -228,31 +229,22 @@ def __call__( "chunk_shift must be set if second pass over the data is selected" ) - # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] - # Pass through the whole provided dl1 data + # Pass through the whole provided dl1 table if self.second_pass: aggregated_stats = aggregator( - table=dl1_table[tel_id], col_name=col_name, chunk_shift=None + table=table[tel_id], + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=None, ) else: aggregated_stats = aggregator( - table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift + table=table[tel_id], + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) # Detect faulty pixels with mutiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) @@ -299,16 +291,17 @@ def __call__( slice_start = max(0, slice_start) + self.chunk_shift # Set the end of the slice to the last element of the dl1 table if out of bound # and subtract one ``chunk_shift``. - slice_end = min( - len(dl1_table[tel_id]) - 1, chunk_size * (index + 2) - ) - (self.chunk_shift - 1) + slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) # Slice the dl1 table according to the previously caluclated start and end. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] + table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. aggregated_stats_secondpass = aggregator( - table=dl1_table_sliced, + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=self.chunk_shift, ) @@ -328,7 +321,9 @@ def __call__( ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["outlier_mask"] = ( + outlier_mask_secondpass + ) # Stack the aggregated statistics of the second pass to the first pass aggregated_stats = vstack( From 61936e5577ac0074772479e1ae486054574f4b5b Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 18:55:03 +0200 Subject: [PATCH 085/221] removed writing part writing should be also done in the tool outside of this component --- src/ctapipe/monitoring/calculator.py | 55 ++++++---------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index f93a88e0b78..d3462aafb6a 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -3,25 +3,21 @@ calculate the montoring data for the camera calibration. """ -import pathlib from abc import abstractmethod import numpy as np -from astropy.table import vstack +from astropy.table import Table, vstack from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Bool, - CaselessStrEnum, ComponentName, Dict, Float, Int, List, - Path, TelescopeParameter, ) -from ctapipe.io import write_table from ctapipe.monitoring.aggregator import StatisticsAggregator from ctapipe.monitoring.outlier import OutlierDetector @@ -30,10 +26,6 @@ "StatisticsCalculator", ] -PEDESTAL_GROUP = "/dl0/monitoring/telescope/pedestal" -FLATFIELD_GROUP = "/dl0/monitoring/telescope/flatfield" -TIMECALIB_GROUP = "/dl0/monitoring/telescope/time_calibration" - class CalibrationCalculator(TelescopeComponent): """ @@ -50,12 +42,6 @@ class CalibrationCalculator(TelescopeComponent): The type of StatisticsAggregator to be used for aggregating statistics. outlier_detector_list : list of dict List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. - calibration_type : ctapipe.core.traits.CaselessStrEnum - The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. - output_path : ctapipe.core.traits.Path - The output filename where the monitoring data will be stored. - overwrite : ctapipe.core.traits.Bool - Whether to overwrite the output file if it exists. """ stats_aggregator_type = TelescopeParameter( @@ -77,18 +63,6 @@ class CalibrationCalculator(TelescopeComponent): ), ).tag(config=True) - calibration_type = CaselessStrEnum( - ["pedestal", "flatfield", "time_calibration"], - allow_none=False, - help="Set type of calibration which is needed to properly store the monitoring data", - ).tag(config=True) - - output_path = Path( - help="Output filename", default_value=pathlib.Path("monitoring.camcalib.h5") - ).tag(config=True) - - overwrite = Bool(help="Overwrite output file if it exists").tag(config=True) - def __init__( self, subarray, @@ -118,12 +92,6 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.group = { - "pedestal": PEDESTAL_GROUP, - "flatfield": FLATFIELD_GROUP, - "time_calibration": TIMECALIB_GROUP, - } - # Initialize the instances of StatisticsAggregator self.stats_aggregator = {} if stats_aggregator is None: @@ -150,7 +118,7 @@ def __init__( ) @abstractmethod - def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): + def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: """ Calculate the monitoring data for a given set of events. @@ -168,6 +136,11 @@ def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): Telescope ID for which the calibration is being performed col_name : str Column name in the table from which the statistics will be aggregated + + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics and their outlier masks """ @@ -178,7 +151,7 @@ class StatisticsCalculator(CalibrationCalculator): This class inherits from CalibrationCalculator and is responsible for calculating various statistics from calibration events, such as pedestal and flat-field data. It aggregates statistics, detects outliers, - handles faulty data chunks, and stores the monitoring data. + handles faulty data chunks. The default option is to conduct only one pass over the data with non-overlapping chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. Two passes over the data, set via the ``second_pass``-flag, can be conducted @@ -221,7 +194,7 @@ def __call__( masked_pixels_of_sample, tel_id, col_name="image", - ): + ) -> Table: # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: @@ -335,11 +308,5 @@ def __call__( self.log.info( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." ) - - # Write the aggregated statistics and their outlier mask to the output file - write_table( - aggregated_stats, - self.output_path, - f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", - overwrite=self.overwrite, - ) + # Return the aggregated statistics and their outlier masks + return aggregated_stats \ No newline at end of file From 52ecd8a385316176f180f2584d336a9ab973da84 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 10:02:30 +0200 Subject: [PATCH 086/221] add unit tests for calculators --- src/ctapipe/monitoring/aggregator.py | 4 +- src/ctapipe/monitoring/calculator.py | 46 ++++---- .../monitoring/tests/test_calculator.py | 106 ++++++++++++++++++ 3 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 src/ctapipe/monitoring/tests/test_calculator.py diff --git a/src/ctapipe/monitoring/aggregator.py b/src/ctapipe/monitoring/aggregator.py index 5f2e5933bbf..862f0758041 100644 --- a/src/ctapipe/monitoring/aggregator.py +++ b/src/ctapipe/monitoring/aggregator.py @@ -59,8 +59,8 @@ def __call__( Parameters ---------- table : astropy.table.Table - table with images of shape (n_images, n_channels, n_pix) - and timestamps of shape (n_images, ) + table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) masked_pixels_of_sample : ndarray, optional boolean array of masked pixels of shape (n_pix, ) that are not available for processing chunk_shift : int, optional diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index d3462aafb6a..750e44cb75f 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -53,7 +53,7 @@ class CalibrationCalculator(TelescopeComponent): ).tag(config=True) outlier_detector_list = List( - trait=Dict, + trait=Dict(), default_value=None, allow_none=True, help=( @@ -118,7 +118,9 @@ def __init__( ) @abstractmethod - def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: + def __call__( + self, table, tel_id, masked_pixels_of_sample=None, col_name="image" + ) -> Table: """ Calculate the monitoring data for a given set of events. @@ -128,15 +130,15 @@ def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: Parameters ---------- table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix) - and timestamps of shape (n_images, ) - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) tel_id : int Telescope ID for which the calibration is being performed + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing col_name : str Column name in the table from which the statistics will be aggregated - + Returns ------- astropy.table.Table @@ -191,8 +193,8 @@ class StatisticsCalculator(CalibrationCalculator): def __call__( self, table, - masked_pixels_of_sample, tel_id, + masked_pixels_of_sample=None, col_name="image", ) -> Table: @@ -207,20 +209,21 @@ def __call__( # Pass through the whole provided dl1 table if self.second_pass: aggregated_stats = aggregator( - table=table[tel_id], + table=table, masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=None, ) else: aggregated_stats = aggregator( - table=table[tel_id], + table=table, masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=self.chunk_shift, ) + # Detect faulty pixels with mutiple instances of ``OutlierDetector`` - outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) + outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( outlier_mask, @@ -251,7 +254,7 @@ def __call__( for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.warning( - f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" + f"Faulty chunk ({int(faulty_pixels_percentage[index])}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( @@ -272,16 +275,19 @@ def __call__( # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) + # Checking for the length of the sliced table to be greater than he chunk_size + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass[0]["mean"], dtype=bool + aggregated_stats_secondpass["mean"], dtype=bool ) for ( aggregated_val, @@ -309,4 +315,4 @@ def __call__( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." ) # Return the aggregated statistics and their outlier masks - return aggregated_stats \ No newline at end of file + return aggregated_stats diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py new file mode 100644 index 00000000000..2878e6d4a0f --- /dev/null +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -0,0 +1,106 @@ +""" +Tests for CalibrationCalculator and related functions +""" + +import numpy as np +from astropy.table import Table +from astropy.time import Time +from traitlets.config.loader import Config + +from ctapipe.monitoring.aggregator import PlainAggregator +from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator + + +def test_onepass_calculator(example_subarray): + """test basic 'one pass' functionality of the StatisticsCalculator""" + + # Create dummy data for testing + times = Time( + np.linspace(60117.911, 60117.9258, num=5000), scale="tai", format="mjd" + ) + event_ids = np.linspace(35, 725000, num=5000, dtype=int) + rng = np.random.default_rng(0) + charge_data = rng.normal(77.0, 10.0, size=(5000, 2, 1855)) + # Create tables + charge_table = Table( + [times, event_ids, charge_data], + names=("time_mono", "event_id", "image"), + ) + # Initialize the aggregators and calculators + chunk_size = 500 + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + calculator = CalibrationCalculator.from_name( + name="StatisticsCalculator", + subarray=example_subarray, + stats_aggregator=aggregator, + ) + calculator_chunk_shift = StatisticsCalculator( + subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=250 + ) + # Compute the statistical values + stats = calculator(table=charge_table, tel_id=1) + stats_chunk_shift = calculator_chunk_shift(table=charge_table, tel_id=1) + + # Check if the calculated statistical values are reasonable + # for a camera with two gain channels + np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) + np.testing.assert_allclose(stats[1]["median"], 77.0, atol=2.5) + np.testing.assert_allclose(stats[0]["std"], 10.0, atol=2.5) + # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows + assert len(stats) * 2 == len(stats_chunk_shift) + 1 + +def test_secondpass_calculator(example_subarray): + """test the chunk shift option and the boundary case for the last chunk""" + + # Create dummy data for testing + times = Time( + np.linspace(60117.911, 60117.9258, num=5500), scale="tai", format="mjd" + ) + event_ids = np.linspace(35, 725000, num=5500, dtype=int) + rng = np.random.default_rng(0) + ped_data = rng.normal(2.0, 5.0, size=(5500, 2, 1855)) + # Create table + ped_table = Table( + [times, event_ids, ped_data], + names=("time_mono", "event_id", "image"), + ) + # Create configuration + config = Config( + { + "StatisticsCalculator": { + "stats_aggregator_type": [ + ("id", 1, "SigmaClippingAggregator"), + ], + "outlier_detector_list": [ + { + "apply_to": "mean", + "name": "StdOutlierDetector", + "validity_range": [-2.0, 2.0], + }, + { + "apply_to": "median", + "name": "StdOutlierDetector", + "validity_range": [-3.0, 3.0], + }, + { + "apply_to": "std", + "name": "RangeOutlierDetector", + "validity_range": [2.0, 8.0], + }, + ], + "chunk_shift": 100, + "second_pass": True, + "faulty_pixels_threshold": 1.0, + }, + "SigmaClippingAggregator": { + "chunk_size": 500, + }, + } + ) + # Initialize the calculator from config + calculator = StatisticsCalculator(subarray=example_subarray, config=config) + # Compute aggregated statistic values + stats = calculator(ped_table, 1, col_name="image") + # Check if the second pass was activated + assert len(stats) > 20 + From 4ffd39ed815538aafdef4acda34a5e4a634ae174 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 10:06:18 +0200 Subject: [PATCH 087/221] add unit tests for calculators --- src/ctapipe/monitoring/calculator.py | 27 +++++++++---------- .../monitoring/tests/test_calculator.py | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 750e44cb75f..5249ab49a14 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -108,13 +108,13 @@ def __init__( self.outlier_detectors = {} if self.outlier_detector_list is not None: for outlier_detector in self.outlier_detector_list: - self.outlier_detectors[outlier_detector["apply_to"]] = ( - OutlierDetector.from_name( - name=outlier_detector["name"], - validity_range=outlier_detector["validity_range"], - subarray=self.subarray, - parent=self, - ) + self.outlier_detectors[ + outlier_detector["apply_to"] + ] = OutlierDetector.from_name( + name=outlier_detector["name"], + validity_range=outlier_detector["validity_range"], + subarray=self.subarray, + parent=self, ) @abstractmethod @@ -197,7 +197,6 @@ def __call__( masked_pixels_of_sample=None, col_name="image", ) -> Table: - # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: raise ValueError( @@ -222,7 +221,7 @@ def __call__( chunk_shift=self.chunk_shift, ) - # Detect faulty pixels with mutiple instances of ``OutlierDetector`` + # Detect faulty pixels with multiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( @@ -270,7 +269,7 @@ def __call__( slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( self.chunk_shift - 1 ) - # Slice the dl1 table according to the previously caluclated start and end. + # Slice the dl1 table according to the previously calculated start and end. table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift @@ -285,7 +284,7 @@ def __call__( chunk_shift=self.chunk_shift, ) - # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass outlier_mask_secondpass = np.zeros_like( aggregated_stats_secondpass["mean"], dtype=bool ) @@ -300,9 +299,9 @@ def __call__( ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = ( - outlier_mask_secondpass - ) + aggregated_stats_secondpass[ + "outlier_mask" + ] = outlier_mask_secondpass # Stack the aggregated statistics of the second pass to the first pass aggregated_stats = vstack( diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 2878e6d4a0f..08c818de0af 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -49,6 +49,7 @@ def test_onepass_calculator(example_subarray): # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows assert len(stats) * 2 == len(stats_chunk_shift) + 1 + def test_secondpass_calculator(example_subarray): """test the chunk shift option and the boundary case for the last chunk""" @@ -103,4 +104,3 @@ def test_secondpass_calculator(example_subarray): stats = calculator(ped_table, 1, col_name="image") # Check if the second pass was activated assert len(stats) > 20 - From c404a4a90cd1c77d2731b0dffb46209638a2d9f4 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 15:38:53 +0200 Subject: [PATCH 088/221] split __call__ function into two function for the first and second pass second pass also has an argument to pass the list of faulty/valid chunks which can be a logical_and from multiple first passes --- src/ctapipe/monitoring/calculator.py | 326 ++++++++++-------- .../monitoring/tests/test_calculator.py | 67 ++-- 2 files changed, 220 insertions(+), 173 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 5249ab49a14..9359e0b1689 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -3,14 +3,11 @@ calculate the montoring data for the camera calibration. """ -from abc import abstractmethod - import numpy as np -from astropy.table import Table, vstack +from astropy.table import Table from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( - Bool, ComponentName, Dict, Float, @@ -117,66 +114,30 @@ def __init__( parent=self, ) - @abstractmethod - def __call__( - self, table, tel_id, masked_pixels_of_sample=None, col_name="image" - ) -> Table: - """ - Calculate the monitoring data for a given set of events. - - This method should be implemented by subclasses to perform the specific - calibration calculations required for different types of calibration. - - Parameters - ---------- - table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and - timestamps of shape (n_images, ) - tel_id : int - Telescope ID for which the calibration is being performed - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing - col_name : str - Column name in the table from which the statistics will be aggregated - - Returns - ------- - astropy.table.Table - Table containing the aggregated statistics and their outlier masks - """ - class StatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. - This class inherits from CalibrationCalculator and is responsible for + This class inherits from ``CalibrationCalculator`` and is responsible for calculating various statistics from calibration events, such as pedestal and flat-field data. It aggregates statistics, detects outliers, handles faulty data chunks. - The default option is to conduct only one pass over the data with non-overlapping - chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. - Two passes over the data, set via the ``second_pass``-flag, can be conducted - with a refined shift of the chunk in regions of trouble with a high percentage - of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. + The ``StatisticsCalculator`` holds two functions to conduct two different passes + over the data with and without overlapping chunks. The first pass is conducted + with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` + parameter in the second pass. The second pass over the data is only conducted + in regions of trouble with a high percentage of faulty pixels exceeding + the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( default_value=None, allow_none=True, help=( - "Number of samples to shift the aggregation chunk for the " - "calculation of the statistical values. If second_pass is set, " - "the first pass is conducted without overlapping chunks (chunk_shift=None) " - "and the second pass with a refined shift of the chunk in regions of trouble." - ), - ).tag(config=True) - - second_pass = Bool( - default_value=False, - help=( - "Set whether to conduct a second pass over the data " - "with a refined shift of the chunk in regions of trouble." + "Number of samples to shift the aggregation chunk for the calculation " + "of the statistical values. Only used in the second_pass(), since the " + "first_pass() is conducted without overlapping chunks (chunk_shift=None)." ), ).tag(config=True) @@ -185,42 +146,52 @@ class StatisticsCalculator(CalibrationCalculator): allow_none=True, help=( "Threshold in percentage of faulty pixels over the camera " - "to conduct second pass with a refined shift of the chunk " - "in regions of trouble." + "to identify regions of trouble." ), ).tag(config=True) - def __call__( + def first_pass( self, table, tel_id, masked_pixels_of_sample=None, col_name="image", ) -> Table: - # Check if the chunk_shift is set for second pass mode - if self.second_pass and self.chunk_shift is None: - raise ValueError( - "chunk_shift must be set if second pass over the data is selected" - ) + """ + Calculate the monitoring data for a given set of events with non-overlapping aggregation chunks. + + This method performs the first pass over the provided data table to calculate + various statistics for calibration purposes. The statistics are aggregated with + non-overlapping chunks (``chunk_shift`` set to None), and faulty pixels are detected + using a list of outlier detectors. + + + Parameters + ---------- + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) + tel_id : int + Telescope ID for which the calibration is being performed + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing + col_name : str + Column name in the table from which the statistics will be aggregated + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics, their outlier masks, and the validity of the chunks + """ # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 table - if self.second_pass: - aggregated_stats = aggregator( - table=table, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=None, - ) - else: - aggregated_stats = aggregator( - table=table, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - + aggregated_stats = aggregator( + table=table, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=None, + ) # Detect faulty pixels with multiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): @@ -230,88 +201,143 @@ def __call__( ) # Add the outlier mask to the aggregated statistics aggregated_stats["outlier_mask"] = outlier_mask + # Get valid chunks and add them to the aggregated statistics + aggregated_stats["is_valid"] = self._get_valid_chunks(outlier_mask) + return aggregated_stats + + def second_pass( + self, + table, + valid_chunks, + tel_id, + masked_pixels_of_sample=None, + col_name="image", + ) -> Table: + """ + Conduct a second pass over the data to refine the statistics in regions with a high percentage of faulty pixels. + + This method performs a second pass over the data with a refined shift of the chunk in regions where a high percentage + of faulty pixels were detected during the first pass. Note: Multiple first passes of different calibration events are + performed which may lead to different identification of faulty chunks in rare cases. Therefore a joined list of faulty + chunks is recommended to be passed to the second pass(es) if those different passes use the same ``chunk_size``. + Parameters + ---------- + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and timestamps of shape (n_images, ). + valid_chunks : ndarray + Boolean array indicating the validity of each chunk from the first pass. + Note: This boolean array can be a ``logical_and`` from multiple first passes of different calibration events. + tel_id : int + Telescope ID for which the calibration is being performed. + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing. + col_name : str + Column name in the table from which the statistics will be aggregated. + + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics after the second pass, their outlier masks, and the validity of the chunks. + """ + # Check if the chunk_shift is set for the second pass + if self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is requested" + ) + # Get the aggregator + aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data - if self.second_pass: - # Check if the camera has two gain channels - if outlier_mask.shape[1] == 2: - # Combine the outlier mask of both gain channels - outlier_mask = np.logical_or( - outlier_mask[:, 0, :], - outlier_mask[:, 1, :], + aggregated_stats_secondpass = None + if np.all(valid_chunks): + self.log.info( + "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + ) + else: + chunk_size = aggregator.chunk_size + faulty_chunks_indices = np.where(~valid_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunk detected in the first pass at index '{index}'." ) - # Calculate the fraction of faulty pixels over the camera - faulty_pixels_percentage = ( - np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] - ) * 100.0 - - # Check for faulty chunks if the threshold is exceeded - faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold - if np.any(faulty_chunks): - chunk_size = aggregated_stats["n_events"][0] - faulty_chunks_indices = np.where(faulty_chunks)[0] - for index in faulty_chunks_indices: - # Log information of the faulty chunks - self.log.warning( - f"Faulty chunk ({int(faulty_pixels_percentage[index])}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" - ) - # Calculate the start of the slice depending on whether the previous chunk was faulty or not - slice_start = ( - chunk_size * index - if index - 1 in faulty_chunks_indices - else chunk_size * (index - 1) + # Calculate the start of the slice depending on whether the previous chunk was faulty or not + slice_start = ( + chunk_size * index + if index - 1 in faulty_chunks_indices + else chunk_size * (index - 1) + ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) + # Slice the dl1 table according to the previously calculated start and end. + table_sliced = table[slice_start:slice_end] + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + # Checking for the length of the sliced table to be greater than he chunk_size + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) - # Set the start of the slice to the first element of the dl1 table if out of bound - # and add one ``chunk_shift``. - slice_start = max(0, slice_start) + self.chunk_shift - # Set the end of the slice to the last element of the dl1 table if out of bound - # and subtract one ``chunk_shift``. - slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( - self.chunk_shift - 1 + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) - # Slice the dl1 table according to the previously calculated start and end. - table_sliced = table[slice_start:slice_end] + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) + return aggregated_stats_secondpass + + def _get_valid_chunks(self, outlier_mask): + """ + Identify valid chunks based on the outlier mask. - # Run the stats aggregator on the sliced dl1 table with a chunk_shift - # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than he chunk_size - # since it can be smaller if the last two chunks are faulty. - if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) + This method processes the outlier mask to determine which chunks of data + are considered valid or faulty. A chunk is marked as faulty if the percentage + of outlier pixels exceeds a predefined threshold ``faulty_pixels_threshold``. - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool - ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector( - aggregated_stats_secondpass[aggregated_val] - ), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass[ - "outlier_mask" - ] = outlier_mask_secondpass + Parameters + ---------- + outlier_mask : numpy.ndarray + Boolean array indicating outlier pixels. The shape of the array should + match the shape of the aggregated statistics. - # Stack the aggregated statistics of the second pass to the first pass - aggregated_stats = vstack( - [aggregated_stats, aggregated_stats_secondpass] - ) - # Sort the aggregated statistics based on the starting time - aggregated_stats.sort(["time_start"]) - else: - self.log.info( - "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." - ) - # Return the aggregated statistics and their outlier masks - return aggregated_stats + Returns + ------- + numpy.ndarray + Boolean array where each element indicates whether the corresponding + chunk is valid (True) or faulty (False). + """ + # Check if the camera has two gain channels + if outlier_mask.shape[1] == 2: + # Combine the outlier mask of both gain channels + outlier_mask = np.logical_or( + outlier_mask[:, 0, :], + outlier_mask[:, 1, :], + ) + # Calculate the fraction of faulty pixels over the camera + faulty_pixels_percentage = ( + np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] + ) * 100.0 + # Check for valid chunks if the threshold is not exceeded + valid_chunks = faulty_pixels_percentage < self.faulty_pixels_threshold + return valid_chunks diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 08c818de0af..37e8fb8fff6 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -3,7 +3,7 @@ """ import numpy as np -from astropy.table import Table +from astropy.table import Table, vstack from astropy.time import Time from traitlets.config.loader import Config @@ -11,8 +11,8 @@ from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator -def test_onepass_calculator(example_subarray): - """test basic 'one pass' functionality of the StatisticsCalculator""" +def test_statistics_calculator(example_subarray): + """test basic functionality of the StatisticsCalculator""" # Create dummy data for testing times = Time( @@ -26,31 +26,40 @@ def test_onepass_calculator(example_subarray): [times, event_ids, charge_data], names=("time_mono", "event_id", "image"), ) - # Initialize the aggregators and calculators - chunk_size = 500 - aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + # Initialize the aggregator and calculator + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) calculator = CalibrationCalculator.from_name( name="StatisticsCalculator", subarray=example_subarray, stats_aggregator=aggregator, - ) - calculator_chunk_shift = StatisticsCalculator( - subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=250 + chunk_shift=100, ) # Compute the statistical values - stats = calculator(table=charge_table, tel_id=1) - stats_chunk_shift = calculator_chunk_shift(table=charge_table, tel_id=1) - + stats = calculator.first_pass(table=charge_table, tel_id=1) + # Set all chunks as faulty to aggregate the statistic values with a "global" chunk shift + valid_chunks = np.zeros_like(stats["is_valid"].data, dtype=bool) + # Run the second pass over the data + stats_chunk_shift = calculator.second_pass( + table=charge_table, valid_chunks=valid_chunks, tel_id=1 + ) + # Stack the statistic values from the first and second pass + stats_combined = vstack([stats, stats_chunk_shift]) + # Sort the combined aggregated statistic values by starting time + stats_combined.sort(["time_start"]) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) np.testing.assert_allclose(stats[1]["median"], 77.0, atol=2.5) np.testing.assert_allclose(stats[0]["std"], 10.0, atol=2.5) - # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows - assert len(stats) * 2 == len(stats_chunk_shift) + 1 + np.testing.assert_allclose(stats_chunk_shift[0]["mean"], 77.0, atol=2.5) + np.testing.assert_allclose(stats_chunk_shift[1]["median"], 77.0, atol=2.5) + np.testing.assert_allclose(stats_chunk_shift[0]["std"], 10.0, atol=2.5) + # Check if overlapping chunks of the second pass were aggregated + assert stats_chunk_shift is not None + assert len(stats_combined) > len(stats) -def test_secondpass_calculator(example_subarray): +def test_outlier_detector(example_subarray): """test the chunk shift option and the boundary case for the last chunk""" # Create dummy data for testing @@ -89,18 +98,30 @@ def test_secondpass_calculator(example_subarray): "validity_range": [2.0, 8.0], }, ], - "chunk_shift": 100, - "second_pass": True, - "faulty_pixels_threshold": 1.0, + "chunk_shift": 500, + "faulty_pixels_threshold": 9.0, }, "SigmaClippingAggregator": { - "chunk_size": 500, + "chunk_size": 1000, }, } ) # Initialize the calculator from config calculator = StatisticsCalculator(subarray=example_subarray, config=config) - # Compute aggregated statistic values - stats = calculator(ped_table, 1, col_name="image") - # Check if the second pass was activated - assert len(stats) > 20 + # Run the first pass over the data + stats_first_pass = calculator.first_pass(table=ped_table, tel_id=1) + # Run the second pass over the data + stats_second_pass = calculator.second_pass( + table=ped_table, valid_chunks=stats_first_pass["is_valid"].data, tel_id=1 + ) + stats_combined = vstack([stats_first_pass, stats_second_pass]) + # Sort the combined aggregated statistic values by starting time + stats_combined.sort(["time_start"]) + # Check if overlapping chunks of the second pass were aggregated + assert stats_second_pass is not None + assert len(stats_combined) > len(stats_second_pass) + # Check if the calculated statistical values are reasonable + # for a camera with two gain channels + np.testing.assert_allclose(stats_combined[0]["mean"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_combined[1]["median"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_combined[0]["std"], 5.0, atol=2.5) From a59664e7f44924da08fca5eabdcc642ac22bdf24 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 15:51:20 +0200 Subject: [PATCH 089/221] fix docs and add changelog --- docs/api-reference/monitoring/calculator.rst | 11 +++++++++++ docs/api-reference/monitoring/index.rst | 3 ++- docs/changes/2609.features.rst | 1 + src/ctapipe/monitoring/calculator.py | 4 ++-- 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 docs/api-reference/monitoring/calculator.rst create mode 100644 docs/changes/2609.features.rst diff --git a/docs/api-reference/monitoring/calculator.rst b/docs/api-reference/monitoring/calculator.rst new file mode 100644 index 00000000000..93a1c1ec861 --- /dev/null +++ b/docs/api-reference/monitoring/calculator.rst @@ -0,0 +1,11 @@ +.. _calibration_calculator: + +********************** +Calibration Calculator +********************** + + +Reference/API +============= + +.. automodapi:: ctapipe.monitoring.calculator diff --git a/docs/api-reference/monitoring/index.rst b/docs/api-reference/monitoring/index.rst index 51268d35fc8..216a7ddc7d6 100644 --- a/docs/api-reference/monitoring/index.rst +++ b/docs/api-reference/monitoring/index.rst @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment. -Currently, only code related to :ref:`stats_aggregator` and :ref:`outlier_detector` is implemented here. +Currently, code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. Submodules @@ -21,6 +21,7 @@ Submodules :glob: aggregator + calculator outlier diff --git a/docs/changes/2609.features.rst b/docs/changes/2609.features.rst new file mode 100644 index 00000000000..fac55b285f6 --- /dev/null +++ b/docs/changes/2609.features.rst @@ -0,0 +1 @@ +Add calibration calculators which aggregates statistics, detects outliers, handles faulty data chunks. diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 9359e0b1689..89cb5b2c3ed 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -126,7 +126,7 @@ class StatisticsCalculator(CalibrationCalculator): The ``StatisticsCalculator`` holds two functions to conduct two different passes over the data with and without overlapping chunks. The first pass is conducted with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` - parameter in the second pass. The second pass over the data is only conducted + parameter for the second pass. The second pass over the data is only conducted in regions of trouble with a high percentage of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ @@ -279,7 +279,7 @@ def second_pass( table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than he chunk_size + # Checking for the length of the sliced table to be greater than the chunk_size # since it can be smaller if the last two chunks are faulty. if len(table_sliced) > aggregator.chunk_size: aggregated_stats_secondpass = aggregator( From 0ed515ddca00ae596b4667b697519137a1507827 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 27 Aug 2024 10:58:45 +0200 Subject: [PATCH 090/221] removed check for any faulty chunk check should be done before and then the second pass should not be called if only valid chunks are provided --- src/ctapipe/monitoring/calculator.py | 100 +++++++++++++-------------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 89cb5b2c3ed..20da80d7428 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -249,62 +249,56 @@ def second_pass( aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data aggregated_stats_secondpass = None - if np.all(valid_chunks): - self.log.info( - "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + faulty_chunks_indices = np.where(~valid_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunk detected in the first pass at index '{index}'." ) - else: - chunk_size = aggregator.chunk_size - faulty_chunks_indices = np.where(~valid_chunks)[0] - for index in faulty_chunks_indices: - # Log information of the faulty chunks - self.log.warning( - f"Faulty chunk detected in the first pass at index '{index}'." - ) - # Calculate the start of the slice depending on whether the previous chunk was faulty or not - slice_start = ( - chunk_size * index - if index - 1 in faulty_chunks_indices - else chunk_size * (index - 1) - ) - # Set the start of the slice to the first element of the dl1 table if out of bound - # and add one ``chunk_shift``. - slice_start = max(0, slice_start) + self.chunk_shift - # Set the end of the slice to the last element of the dl1 table if out of bound - # and subtract one ``chunk_shift``. - slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( - self.chunk_shift - 1 - ) - # Slice the dl1 table according to the previously calculated start and end. - table_sliced = table[slice_start:slice_end] - # Run the stats aggregator on the sliced dl1 table with a chunk_shift - # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than the chunk_size - # since it can be smaller if the last two chunks are faulty. - if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool + # Calculate the start of the slice depending on whether the previous chunk was faulty or not + slice_start = ( + aggregator.chunk_size * index + if index - 1 in faulty_chunks_indices + else aggregator.chunk_size * (index - 1) + ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min(len(table) - 1, aggregator.chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) + # Slice the dl1 table according to the previously calculated start and end. + table_sliced = table[slice_start:slice_end] + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + # Checking for the length of the sliced table to be greater than the ``chunk_size`` + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass - aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( - outlier_mask_secondpass + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) return aggregated_stats_secondpass def _get_valid_chunks(self, outlier_mask): From 48c686b93bb483d37b74be8b5875b8a076c57970 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 27 Aug 2024 18:08:17 +0200 Subject: [PATCH 091/221] removed base class CalibrationCalculator and only have one StatisticsCalculator class --- docs/api-reference/monitoring/index.rst | 2 +- src/ctapipe/monitoring/aggregator.py | 2 +- src/ctapipe/monitoring/calculator.py | 86 +++++++------------ .../monitoring/tests/test_calculator.py | 5 +- 4 files changed, 37 insertions(+), 58 deletions(-) diff --git a/docs/api-reference/monitoring/index.rst b/docs/api-reference/monitoring/index.rst index 216a7ddc7d6..11ae54bd7e0 100644 --- a/docs/api-reference/monitoring/index.rst +++ b/docs/api-reference/monitoring/index.rst @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment. -Currently, code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. +Code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. Submodules diff --git a/src/ctapipe/monitoring/aggregator.py b/src/ctapipe/monitoring/aggregator.py index 862f0758041..a6c8be44c5f 100644 --- a/src/ctapipe/monitoring/aggregator.py +++ b/src/ctapipe/monitoring/aggregator.py @@ -51,7 +51,7 @@ def __call__( and call the relevant function of the particular aggregator to compute aggregated statistic values. The chunks are generated in a way that ensures they do not overflow the bounds of the table. - If ``chunk_shift`` is None, chunks will not overlap, but the last chunk is ensured to be - of size `chunk_size`, even if it means the last two chunks will overlap. + of size ``chunk_size``, even if it means the last two chunks will overlap. - If ``chunk_shift`` is provided, it will determine the number of samples to shift between the start of consecutive chunks resulting in an overlap of chunks. Chunks that overflows the bounds of the table are not considered. diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 20da80d7428..a3adc01d370 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -1,5 +1,5 @@ """ -Definition of the ``CalibrationCalculator`` classes, providing all steps needed to +Definition of the ``StatisticsCalculator`` class, providing all steps needed to calculate the montoring data for the camera calibration. """ @@ -19,26 +19,22 @@ from ctapipe.monitoring.outlier import OutlierDetector __all__ = [ - "CalibrationCalculator", "StatisticsCalculator", ] -class CalibrationCalculator(TelescopeComponent): +class StatisticsCalculator(TelescopeComponent): """ - Base component for calibration calculators. - - This class provides the foundational methods and attributes for - calculating camera-related monitoring data. It is designed - to be extended by specific calibration calculators that implement - the required methods for different types of calibration. + Component to calculate statistics from calibration events. - Attributes - ---------- - stats_aggregator_type : ctapipe.core.traits.TelescopeParameter - The type of StatisticsAggregator to be used for aggregating statistics. - outlier_detector_list : list of dict - List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. + The ``StatisticsCalculator`` is responsible for calculating various statistics from + calibration events, such as pedestal and flat-field data. It aggregates statistics, + detects outliers, and handles faulty data periods. + This class holds two functions to conduct two different passes over the data with and without + overlapping aggregation chunks. The first pass is conducted with non-overlapping chunks, + while overlapping chunks can be set by the ``chunk_shift`` parameter for the second pass. + The second pass over the data is only conducted in regions of trouble with a high percentage + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ stats_aggregator_type = TelescopeParameter( @@ -56,7 +52,27 @@ class CalibrationCalculator(TelescopeComponent): help=( "List of dicts containing the name of the OutlierDetector subclass to be used, " "the aggregated statistic value to which the detector should be applied, " - "and the validity range of the detector." + "and the validity range of the detector. " + "E.g. ``[{'apply_to': 'std', 'name': 'RangeOutlierDetector', 'validity_range': [2.0, 8.0]},]``." + ), + ).tag(config=True) + + chunk_shift = Int( + default_value=None, + allow_none=True, + help=( + "Number of samples to shift the aggregation chunk for the calculation " + "of the statistical values. Only used in the second_pass(), since the " + "first_pass() is conducted with non-overlapping chunks (chunk_shift=None)." + ), + ).tag(config=True) + + faulty_pixels_threshold = Float( + default_value=10.0, + allow_none=True, + help=( + "Threshold in percentage of faulty pixels over the camera " + "to identify regions of trouble." ), ).tag(config=True) @@ -114,42 +130,6 @@ def __init__( parent=self, ) - -class StatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - - This class inherits from ``CalibrationCalculator`` and is responsible for - calculating various statistics from calibration events, such as pedestal - and flat-field data. It aggregates statistics, detects outliers, - handles faulty data chunks. - The ``StatisticsCalculator`` holds two functions to conduct two different passes - over the data with and without overlapping chunks. The first pass is conducted - with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` - parameter for the second pass. The second pass over the data is only conducted - in regions of trouble with a high percentage of faulty pixels exceeding - the threshold ``faulty_pixels_threshold``. - """ - - chunk_shift = Int( - default_value=None, - allow_none=True, - help=( - "Number of samples to shift the aggregation chunk for the calculation " - "of the statistical values. Only used in the second_pass(), since the " - "first_pass() is conducted without overlapping chunks (chunk_shift=None)." - ), - ).tag(config=True) - - faulty_pixels_threshold = Float( - default_value=10.0, - allow_none=True, - help=( - "Threshold in percentage of faulty pixels over the camera " - "to identify regions of trouble." - ), - ).tag(config=True) - def first_pass( self, table, @@ -252,7 +232,7 @@ def second_pass( faulty_chunks_indices = np.where(~valid_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks - self.log.warning( + self.log.info( f"Faulty chunk detected in the first pass at index '{index}'." ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 37e8fb8fff6..876bfc3955c 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -8,7 +8,7 @@ from traitlets.config.loader import Config from ctapipe.monitoring.aggregator import PlainAggregator -from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator +from ctapipe.monitoring.calculator import StatisticsCalculator def test_statistics_calculator(example_subarray): @@ -28,8 +28,7 @@ def test_statistics_calculator(example_subarray): ) # Initialize the aggregator and calculator aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) - calculator = CalibrationCalculator.from_name( - name="StatisticsCalculator", + calculator = StatisticsCalculator( subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=100, From 3bae046f1350b6cbc0ca06f7b59a0de829f8ecd7 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 09:17:49 +0200 Subject: [PATCH 092/221] fix fstring in logging --- src/ctapipe/monitoring/calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index a3adc01d370..b58c4259b70 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -233,7 +233,7 @@ def second_pass( for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.info( - f"Faulty chunk detected in the first pass at index '{index}'." + "Faulty chunk detected in the first pass at index '%s'.", index ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( From aa5666af97efa8995007ef67d46ead2baec727bf Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 17:52:56 +0200 Subject: [PATCH 093/221] bug fix aggregated stats of the second pass we simply overwritten rather than append and vstacked() unit tests are improved to properly test for the correct number of chunks --- src/ctapipe/monitoring/calculator.py | 58 +++++++++++-------- .../monitoring/tests/test_calculator.py | 53 ++++++++++++----- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index b58c4259b70..d95041cf75c 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -4,7 +4,7 @@ """ import numpy as np -from astropy.table import Table +from astropy.table import Table, vstack from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( @@ -225,10 +225,15 @@ def second_pass( raise ValueError( "chunk_shift must be set if second pass over the data is requested" ) + # Check if at least one chunk is faulty + if np.all(valid_chunks): + raise ValueError( + "All chunks are valid. The second pass over the data is redundant." + ) # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data - aggregated_stats_secondpass = None + aggregated_stats_secondpass = [] faulty_chunks_indices = np.where(~valid_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks @@ -254,31 +259,36 @@ def second_pass( # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. # Checking for the length of the sliced table to be greater than the ``chunk_size`` - # since it can be smaller if the last two chunks are faulty. + # since it can be smaller if the last two chunks are faulty. Note: The two last chunks + # can be overlapping during the first pass, so we simply ignore them if there are faulty. if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool - ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), + aggregated_stats_secondpass.append( + aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass - aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( - outlier_mask_secondpass + # Stack the aggregated statistics of each faulty chunk + aggregated_stats_secondpass = vstack(aggregated_stats_secondpass) + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) return aggregated_stats_secondpass def _get_valid_chunks(self, outlier_mask): diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 876bfc3955c..520254724ed 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -15,23 +15,26 @@ def test_statistics_calculator(example_subarray): """test basic functionality of the StatisticsCalculator""" # Create dummy data for testing + n_images = 5050 times = Time( - np.linspace(60117.911, 60117.9258, num=5000), scale="tai", format="mjd" + np.linspace(60117.911, 60117.9258, num=n_images), scale="tai", format="mjd" ) - event_ids = np.linspace(35, 725000, num=5000, dtype=int) + event_ids = np.linspace(35, 725000, num=n_images, dtype=int) rng = np.random.default_rng(0) - charge_data = rng.normal(77.0, 10.0, size=(5000, 2, 1855)) + charge_data = rng.normal(77.0, 10.0, size=(n_images, 2, 1855)) # Create tables charge_table = Table( [times, event_ids, charge_data], names=("time_mono", "event_id", "image"), ) # Initialize the aggregator and calculator - aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) + chunk_size = 1000 + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + chunk_shift = 500 calculator = StatisticsCalculator( subarray=example_subarray, stats_aggregator=aggregator, - chunk_shift=100, + chunk_shift=chunk_shift, ) # Compute the statistical values stats = calculator.first_pass(table=charge_table, tel_id=1) @@ -42,9 +45,12 @@ def test_statistics_calculator(example_subarray): table=charge_table, valid_chunks=valid_chunks, tel_id=1 ) # Stack the statistic values from the first and second pass - stats_combined = vstack([stats, stats_chunk_shift]) - # Sort the combined aggregated statistic values by starting time - stats_combined.sort(["time_start"]) + stats_stacked = vstack([stats, stats_chunk_shift]) + # Sort the stacked aggregated statistic values by starting time + stats_stacked.sort(["time_start"]) + print(stats) + print(stats_chunk_shift) + print(stats_stacked) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) @@ -55,7 +61,21 @@ def test_statistics_calculator(example_subarray): np.testing.assert_allclose(stats_chunk_shift[0]["std"], 10.0, atol=2.5) # Check if overlapping chunks of the second pass were aggregated assert stats_chunk_shift is not None - assert len(stats_combined) > len(stats) + # Check if the number of aggregated chunks is correct + # In the first pass, the number of chunks is equal to the + # number of images divided by the chunk size plus one + # overlapping chunk at the end. + expected_len_firstpass = n_images // chunk_size + 1 + assert len(stats) == expected_len_firstpass + # In the second pass, the number of chunks is equal to the + # number of images divided by the chunk shift minus the + # number of chunks in the first pass, since we set all + # chunks to be faulty. + expected_len_secondpass = (n_images // chunk_shift) - expected_len_firstpass + assert len(stats_chunk_shift) == expected_len_secondpass + # The total number of aggregated chunks is the sum of the + # number of chunks in the first and second pass. + assert len(stats_stacked) == expected_len_firstpass + expected_len_secondpass def test_outlier_detector(example_subarray): @@ -113,14 +133,15 @@ def test_outlier_detector(example_subarray): stats_second_pass = calculator.second_pass( table=ped_table, valid_chunks=stats_first_pass["is_valid"].data, tel_id=1 ) - stats_combined = vstack([stats_first_pass, stats_second_pass]) - # Sort the combined aggregated statistic values by starting time - stats_combined.sort(["time_start"]) + # Stack the statistic values from the first and second pass + stats_stacked = vstack([stats_first_pass, stats_second_pass]) + # Sort the stacked aggregated statistic values by starting time + stats_stacked.sort(["time_start"]) # Check if overlapping chunks of the second pass were aggregated assert stats_second_pass is not None - assert len(stats_combined) > len(stats_second_pass) + assert len(stats_stacked) > len(stats_second_pass) # Check if the calculated statistical values are reasonable # for a camera with two gain channels - np.testing.assert_allclose(stats_combined[0]["mean"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_combined[1]["median"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_combined[0]["std"], 5.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[0]["mean"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[1]["median"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[0]["std"], 5.0, atol=2.5) From ef53ebbf3de60af79188e4abf402b2ef25b81482 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 18:00:19 +0200 Subject: [PATCH 094/221] removed print --- src/ctapipe/monitoring/tests/test_calculator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 520254724ed..b64630f2597 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -48,9 +48,6 @@ def test_statistics_calculator(example_subarray): stats_stacked = vstack([stats, stats_chunk_shift]) # Sort the stacked aggregated statistic values by starting time stats_stacked.sort(["time_start"]) - print(stats) - print(stats_chunk_shift) - print(stats_stacked) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) From 3ca72525071b0120888fa6b60d90bfaec877847d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 3 Sep 2024 12:14:05 +0200 Subject: [PATCH 095/221] removed me from CODEOWNERS --- CODEOWNERS | 2 -- 1 file changed, 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 70c53117a07..06c29bc6c78 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,8 +5,6 @@ ctapipe/calib/camera @watsonjj ctapipe/image/extractor.py @watsonjj @HealthyPear -ctapipe/monitoring @TjarkMiener - ctapipe/reco/HillasReconstructor.py @HealthyPear ctapipe/reco/tests/test_HillasReconstructor.py @HealthyPear From c5f385f0ef16f5270fff354854447679054ef296 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 3 Sep 2024 15:53:10 +0200 Subject: [PATCH 096/221] I added a basic star fitter --- src/ctapipe/calib/camera/calibrator.py | 362 +--------------- src/ctapipe/calib/camera/pointing.py | 571 +++++++++++++++++++++++++ src/ctapipe/containers.py | 29 ++ 3 files changed, 601 insertions(+), 361 deletions(-) create mode 100644 src/ctapipe/calib/camera/pointing.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 98f58454f5b..baf3d2f1057 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,42 +2,24 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ - -import pickle -from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np -import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, - Float, - Int, - Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "PointingCalculator", - "CameraCalibrator", -] +__all__ = ["CameraCalibrator"] @cache @@ -65,348 +47,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - - -class PointingCalculator(CalibrationCalculator): - """ - Component to calculate pointing corrections from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - telescope_location: dict - The location of the telescope for which the pointing correction is to be calculated - """ - - telescope_location = Dict( - {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, - help="Telescope location, longitude and latitude should be expressed in deg, " - "elevation - in meters", - ).tag(config=True) - - min_star_prominence = Integer( - 3, - help="Minimal star prominence over the background in terms of " - "NSB variance std deviations", - ).tag(config=True) - - max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" - ).tag(config=True) - - psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), - default_value="ComaModel", - help="Name of the PSFModel Subclass to be used.", - ).tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - **kwargs, - ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self - ) - - self.location = EarthLocation( - lon=self.telescope_location["longitude"] * u.deg, - lat=self.telescope_location["latitude"] * u.deg, - height=self.telescope_location["elevation"] * u.m, - ) - - def __call__(self, input_url, tel_id): - if self._check_req_data(input_url, tel_id, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - self.tel_id = tel_id - - # first get thecamera geometry and pointing for the file and determine what stars we should see - - with EventSource(input_url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) - - stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] - - stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - - # get the time and images from the data - - variance_images = dl1_table["variance_image"] - - time = dl1_table["time"] - - # now calibrate the images - - variance_images = self._calibrate_var_images( - self, variance_images, time, input_url - ) - - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, time, calibration_file): - # So i need to use the interpolator classes to read the calibration data - relative_gains = FlatFieldInterpolator( - calibration_file - ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square(relative_gains(time[i])), - ) - - return var_images - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py new file mode 100644 index 00000000000..2f89159bcdf --- /dev/null +++ b/src/ctapipe/calib/camera/pointing.py @@ -0,0 +1,571 @@ +""" +Definition of the `CameraCalibrator` class, providing all steps needed to apply +calibration and image extraction, as well as supporting algorithms. +""" + +import copy +from functools import cache + +import astropy.units as u +import numpy as np +import Vizier # discuss this dependency with max etc. +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import StatisticsExtractor +from ctapipe.containers import StarContainer +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + ComponentName, + Dict, + Float, + Integer, + TelescopeParameter, +) +from ctapipe.image import tailcuts_clean +from ctapipe.image.psf_model import PSFModel +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader + +__all__ = [ + "PointingCalculator", +] + + +@cache +def _get_pixel_index(n_pixels): + """Cached version of ``np.arange(n_pixels)``""" + return np.arange(n_pixels) + + +def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channel): + broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) + + index = _get_pixel_index(n_pixels) + masks = ( + pixel_status.hardware_failing_pixels, + pixel_status.pedestal_failing_pixels, + pixel_status.flatfield_failing_pixels, + ) + for mask in masks: + if mask is not None: + if selected_gain_channel is not None: + broken_pixels |= mask[selected_gain_channel, index] + else: + broken_pixels |= mask + + return broken_pixels + + +def cart2pol(x, y): + """ + Convert cartesian coordinates to polar + + :param float x: X coordinate [m] + :param float y: Y coordinate [m] + + :return: Tuple (r, φ)[m, rad] + """ + rho = np.sqrt(x**2 + y**2) + phi = np.arctan2(y, x) + return (rho, phi) + + +def pol2cart(rho, phi): + """ + Convert polar coordinates to cartesian + + :param float rho: R coordinate + :param float phi: ¢ coordinate [rad] + + :return: Tuple (x,y)[m, m] + """ + x = rho * np.cos(phi) + y = rho * np.sin(phi) + return (x, y) + + +class PointingCalculator(TelescopeComponent): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + cleaning = Dict( + {"bound_thresh": 750, "pic_thresh": 15000}, help="Image cleaning parameters" + ).tag(config=True) + + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + + psf_model_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="ComaModel", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__( + subarray=subarray, + stats_extractor="Plain", + config=config, + parent=parent, + **kwargs, + ) + + self.psf = PSFModel.from_name( + self.psf_model_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, input_url, tel_id): + self.tel_id = tel_id + + if self._check_req_data(input_url, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + # first get the camera geometry and pointing for the file and determine what stars we should see + + with EventSource(input_url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) # get some pointing to make a list of stars that we expect to see + + self.pointing = self.pointing.transform_to("icrs") + + self.broken_pixels = np.unique(np.where(self.broken_pixels)) + + self.image_size = len( + event.variance_image.image + ) # get the size of images of the camera we are calibrating + + self.stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] # get all stars that could be in the fov + + self.stars_in_fov = self.stars_in_fov[ + self.tars_in_fov["Bmag"] < self.max_star_magnitude + ] # select stars for magnitude to exclude those we would not be able to see + + # get the accumulated variance images + + ( + accumulated_pointing, + accumulated_times, + variance_statistics, + ) = self._get_accumulated_images(input_url) + + accumulated_images = np.array([x.mean for x in variance_statistics]) + + star_pixels = self._get_expected_star_pixels( + accumulated_times, accumulated_pointing + ) + + star_mask = np.ones(self.image_size, dtype=bool) + + star_mask[star_pixels] = False + + # get NSB values + + nsb = np.mean(accumulated_images[star_mask], axis=1) + nsb_std = np.std(accumulated_images[star_mask], axis=1) + + clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) + + reco_stars = [] + + for i, image in enumerate(clean_images): + reco_stars.append([]) + camera_frame = EngineeringCameraFrame( + telescope_pointing=accumulated_pointing[i], + focal_length=self.focal_length, + obstime=accumulated_times[i].utc, + location=self.location, + ) + for star in self.stars_in_fov: + reco_stars[-1].append( + self._fit_star_position( + star, accumulated_times[i], camera_frame, image, nsb_std[i] + ) + ) + + return reco_stars + + # now fit the star locations + + def _check_req_data(self, url, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[self.tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + + def _calibrate_var_images(self, var_images, time, calibration_file): + """ + Calibrate a set of variance images + + Parameters + ---------- + var_images : list + list of variance images + time : list + list of times correxponding to the variance images + calibration_file : str + name of the file where the calibration data can be found + """ + # So i need to use the interpolator classes to read the calibration data + relative_gains = FlatFieldInterpolator( + calibration_file + ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, + np.square(relative_gains(time[i])), + ) + + return var_images + + def _get_expected_star_pixels(self, time_list, pointing_list): + """ + Determine which in which pixels stars are expected for a series of images + + Parameters + ---------- + time_list : list + list of time values where the images were capturedd + pointing_list : list + list of pointing values for the images + """ + + res = [] + + for pointing, time in zip( + pointing_list, time_list + ): # loop over time and pointing of images + temp = [] + + camera_frame = EngineeringCameraFrame( + telescope_pointing=pointing, + focal_length=self.focal_length, + obstime=time.utc, + location=self.location, + ) # get the engineering camera frame for the pointing + + for star in self.stars_in_fov: + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + expected_central_pixel = self.camera_geometry.transform_to( + camera_frame + ).position_to_pix_index( + star_coords.x, star_coords.y + ) # get where the star should be + cluster = copy.deepcopy( + self.camera_geometry.neighbors[expected_central_pixel] + ) # get the neighborhood of the star + cluster_corona = [] + + for pixel_index in cluster: + cluster_corona.extend( + copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) + ) # and add another layer of pixels to be sure + + cluster.extend(cluster_corona) + cluster.append(expected_central_pixel) + temp.extend(list(set(cluster))) + + res.append(temp) + + return res + + def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + + rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) + + if phi < 0: + phi = phi + 2 * np.pi + + star_container = StarContainer( + label=star["NOMAD1"], + magnitude=star["Bmag"], + expected_x=star_coords.x, + expected_y=star_coords.y, + expected_r=rho * u.m, + expected_phi=phi * u.rad, + timestamp=timestamp, + ) + + current_geometry = self.camera_geometry.transform_to(camera_frame) + + hit_pdf = self._get_star_pdf(star, current_geometry) + cluster = np.where(hit_pdf > self.pdf_percentile_limit * np.sum(hit_pdf)) + + if not np.any(image[cluster] > self.min_star_prominence * nsb_std): + self.log.info("Star %s can not be detected", star["NOMAD1"]) + star.pixels = np.full(self.max_cluster_size, -1) + return star_container + + pad_size = self.max_cluster_size - len(cluster[0]) + if pad_size > 0: + star.pixels = np.pad(cluster[0], (0, pad_size), constant_values=-1) + else: + star.pixels = cluster[0][: self.max_cluster_size] + + self.log.warning( + "Reconstructed cluster is longer than %s, truncated cluster info will " + "be recorded to the output table. Not a big deal, as correct cluster " + "used for position reconstruction.", + self.max_cluster_size, + ) + return star_container + + rs, fs = cart2pol( + current_geometry.pix_x[cluster].to_value(u.m), + current_geometry.pix_y[cluster].to_value(u.m), + ) + + k, r0, sr = self.psf_model.radial_pdf_params + + star_container.reco_r = ( + self.coma_r_shift_correction + * np.average(rs, axis=None, weights=image[cluster], returned=False) + * u.m + ) + + star_container.reco_x = self.coma_r_shift_correction * np.average( + current_geometry.pix_x[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + star_container.reco_y = self.coma_r_shift_correction * np.average( + current_geometry.pix_y[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + _, star_container.reco_phi = cart2pol(star.reco_x, star.reco_y) + + if star_container.reco_phi < 0: + star_container.reco_phi = star.reco_phi + 2 * np.pi * u.rad + + star_container.reco_dx = ( + np.sqrt(np.cov(current_geometry.pix_x[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dy = ( + np.sqrt(np.cov(current_geometry.pix_y[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dr = np.sqrt(np.cov(rs, aweights=hit_pdf[cluster])) * u.m + + _, star_container.reco_dphi = cart2pol( + star_container.reco_dx, star_container.reco_dy + ) + + return star_container + + def _get_accumulated_images(self, input_url): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=self.tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=True, + ) + + # get the trigger type for all images and make a mask + + event_mask = dl1_table["event_type"] == 2 + + # get the pointing for all images and filter for trigger type + + altitude = dl1_table["telescope_pointing_altitude"][event_mask] + azimuth = dl1_table["telescope_pointing_azimuth"][event_mask] + time = dl1_table["time"][event_mask] + + pointing = [ + SkyCoord( + az=x, alt=y, frame="altaz", obstime=z.tai.utc, location=self.location + ) + for x, y, z in zip(azimuth, altitude, time) + ] + + # get the time and images from the data + + variance_images = copy.deepcopy(dl1_table["variance_image"][event_mask]) + + # now make a filter to to reject EAS light and starlight and keep a separate EAS filter + + charge_images = dl1_table["image"][event_mask] + + light_mask = [ + tailcuts_clean( + self.camera_geometry, + x, + picture_thresh=self.cleaning["pic_thresh"], + boundary_thresh=self.cleaning["bound_thresh"], + ) + for x in charge_images + ] + + shower_mask = copy.deepcopy(light_mask) + + star_pixels = self._get_expected_star_pixels(time, pointing) + + light_mask[:, star_pixels] = True + + if self.broken_pixels is not None: + light_mask[:, self.broken_pixels] = True + + # calculate the average variance in viable pixels and replace the values where there is EAS light + + mean_variance = np.mean(variance_images[~light_mask]) + + variance_images[shower_mask] = mean_variance + + # now calibrate the images + + variance_images = self._calibrate_var_images( + self, variance_images, time, input_url + ) + + # Get the average variance across the data to + + # then turn it into a table that the extractor can read + variance_image_table = QTable([time, variance_images], names=["time", "image"]) + + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[self.tel_id]] + + # get the cumulative variance images using the statistics extractor and return the value + + variance_statistics = extractor(variance_image_table) + + accumulated_times = np.array([x.validity_start for x in variance_statistics]) + + # calculate where stars might be + + accumulated_pointing = np.array( + [x for x in pointing if pointing.time in accumulated_times] + ) + + return (accumulated_pointing, accumulated_times, variance_statistics) + + def _get_star_pdf(self, star, current_geometry): + image = np.zeros(self.image_size) + + r0 = star.expected_r.to_value(u.m) + f0 = star.expected_phi.to_value(u.rad) + + self.psf_model.update_model_parameters(r0, f0) + + dr = ( + self.pdf_bin_size + * np.rad2deg(np.arctan(1 / self.focal_length.to_value(u.m))) + / 3600.0 + ) + r = np.linspace( + r0 - dr * self.n_pdf_bins / 2.0, + r0 + dr * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + df = np.deg2rad(self.pdf_bin_size / 3600.0) * 100 + f = np.linspace( + f0 - df * self.n_pdf_bins / 2.0, + f0 + df * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + + for r_ in r: + for f_ in f: + val = self.psf_model.pdf(r_, f_) * dr * df + x, y = pol2cart(r_, f_) + pixelN = current_geometry.position_to_pix_index(x * u.m, y * u.m) + if pixelN != -1: + image[pixelN] += val + + return image diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 311142311a3..489042a1953 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -64,6 +64,7 @@ "ObservationBlockContainer", "ObservingMode", "ObservationBlockState", + "StarContainer", ] @@ -1529,3 +1530,31 @@ class ObservationBlockContainer(Container): scheduled_start_time = Field(NAN_TIME, "expected start time from scheduler") actual_start_time = Field(NAN_TIME, "true start time") actual_duration = Field(nan * u.min, "true duration", unit=u.min) + + +class StarContainer(Container): + "Stores information about a star in the field of view of a camera." + + label = Field("", "Star label", dtype=np.str_) + magnitude = Field(-1, "Star magnitude") + expected_x = Field(np.nan * u.m, "Expected star position (x)", unit=u.m) + expected_y = Field(np.nan * u.m, "Expected star position (y)", unit=u.m) + + expected_r = Field(np.nan * u.m, "Expected star position (r)", unit=u.m) + expected_phi = Field(np.nan * u.rad, "Expected star position (phi)", unit=u.rad) + + reco_x = Field(np.nan * u.m, "Reconstructed star position (x)", unit=u.m) + reco_y = Field(np.nan * u.m, "Reconstructed star position (y)", unit=u.m) + reco_dx = Field(np.nan * u.m, "Reconstructed star position error (x)", unit=u.m) + reco_dy = Field(np.nan * u.m, "Reconstructed star position error (y)", unit=u.m) + + reco_r = Field(np.nan * u.m, "Reconstructed star position (r)", unit=u.m) + reco_phi = Field(np.nan * u.rad, "Reconstructed star position (phi)", unit=u.rad) + reco_dr = Field(np.nan * u.m, "Reconstructed star position error (r)", unit=u.m) + reco_dphi = Field( + np.nan * u.rad, "Reconstructed star position error (phi)", unit=u.rad + ) + + timestamp = Field(NAN_TIME, "Reconstruction timestamp") + + pixels = Field(np.full(20, -1), "List of star pixel ids", dtype=np.int_, ndim=1) From 2756a38e492b6863a0a2962131b1fc8d96ed7027 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 12 Sep 2024 11:08:20 +0200 Subject: [PATCH 097/221] Added the fitting --- src/ctapipe/calib/camera/pointing.py | 326 ++++++++++++++++++++++++++- 1 file changed, 322 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 2f89159bcdf..77105194382 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -8,9 +8,12 @@ import astropy.units as u import numpy as np +import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord from astropy.table import QTable +from astropy.time import Time +from scipy.odr import ODR, Model, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer @@ -85,6 +88,87 @@ def pol2cart(rho, phi): return (x, y) +class StarTracker: + """ + Utility class to provide the position of the star in the telescope's camera frame coordinates at a given time + """ + + def __init__( + self, + star_label, + star_coordinates, + telescope_location, + telescope_focal_length, + telescope_pointing, + observed_wavelength, + relative_humidity, + temperature, + pressure, + pointing_label=None, + ): + """ + Constructor + + :param str star_label: Star label + :param SkyCoord star_coordinates: Star coordinates in ICRS frame + :param EarthLocation telescope_location: Telescope location coordinates + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str pointing_label: Pointing label + """ + self.star_label = star_label + self.star_coordinates_icrs = star_coordinates + self.telescope_location = telescope_location + self.telescope_focal_length = telescope_focal_length + self.telescope_pointing = telescope_pointing + self.obswl = observed_wavelength + self.relative_humidity = relative_humidity + self.temperature = temperature + self.pressure = pressure + self.pointing_label = pointing_label + + def position_in_camera_frame(self, timestamp, pointing=None, focal_correction=0): + """ + Calculates star position in the engineering camera frame + + :param astropy.Time timestamp: Timestamp of the observation + :param SkyCoord pointing: Current telescope pointing in ICRS frame + :param float focal_correction: Correction to the focal length of the telescope. Float, should be provided in meters + + :return: Pair (float, float) of star's (x,y) coordinates in the engineering camera frame in meters + """ + # If no telescope pointing is provided, use the telescope pointing, provided + # during the class member initialization + if pointing is None: + pointing = self.telescope_pointing + # Determine current telescope pointing in AltAz + altaz_pointing = pointing.transform_to( + AltAz( + obstime=timestamp, + location=self.telescope_location, + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + ) + # Create current camera frame + camera_frame = EngineeringCameraFrame( + telescope_pointing=altaz_pointing, + focal_length=self.telescope_focal_length + focal_correction * u.m, + obstime=timestamp, + location=self.telescope_location, + ) + # Calculate the star's coordinates in the current camera frame + star_coords_camera = self.star_coordinates_icrs.transform_to(camera_frame) + return (star_coords_camera.x.to_value(), star_coords_camera.y.to_value()) + + class PointingCalculator(TelescopeComponent): """ Component to calculate pointing corrections from interleaved skyfield events. @@ -103,6 +187,12 @@ class PointingCalculator(TelescopeComponent): "elevation - in meters", ).tag(config=True) + observed_wavelength = Float( + 0.35, + help="Observed star light wavelength in microns" + "(convolution of blackbody spectrum with camera sensitivity)", + ).tag(config=True) + min_star_prominence = Integer( 3, help="Minimal star prominence over the background in terms of " @@ -128,6 +218,11 @@ class PointingCalculator(TelescopeComponent): help="Name of the PSFModel Subclass to be used.", ).tag(config=True) + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + def __init__( self, subarray, @@ -196,6 +291,8 @@ def __call__(self, input_url, tel_id): self.tars_in_fov["Bmag"] < self.max_star_magnitude ] # select stars for magnitude to exclude those we would not be able to see + star_labels = [x.label for x in self.stars_in_fov] + # get the accumulated variance images ( @@ -240,7 +337,17 @@ def __call__(self, input_url, tel_id): return reco_stars - # now fit the star locations + # now fit the pointing correction. + fitter = Pointing_Fitter( + star_labels, + self.pointing, + self.location, + self.focal_length, + self.observed_wavelength, + self.meteo_parameters, + ) + + fitter.fit(accumulated_pointing) def _check_req_data(self, url, calibration_type): """ @@ -299,7 +406,7 @@ def _get_expected_star_pixels(self, time_list, pointing_list): Parameters ---------- time_list : list - list of time values where the images were capturedd + list of time values where the images were captured pointing_list : list list of pointing values for the images """ @@ -346,10 +453,13 @@ def _get_expected_star_pixels(self, time_list, pointing_list): return res - def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + def _fit_star_position( + self, star, timestamp, camera_frame, image, nsb_std, current_pointing + ): star_coords = SkyCoord( star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" ) + star_coords = star_coords.transform_to(camera_frame) rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) @@ -569,3 +679,211 @@ def _get_star_pdf(self, star, current_geometry): image[pixelN] += val return image + + +class Pointing_Fitter: + """ + Pointing correction fitter + """ + + def __init__( + self, + stars, + times, + telescope_pointing, + telescope_location, + focal_length, + observed_wavelength, + meteo_params, + fit_grid="polar", + ): + """ + Constructor + + :param list stars: List of lists of star containers the first dimension is supposed to be time + :param list time: List of time values for the when the star locations were fitted + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param EarthLocation telescope_location: Telescope location + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str fit_grid: Coordinate system grid to use. Either polar or cartesian + """ + self.star_containers = stars + self.times = times + self.telescope_pointing = telescope_pointing + self.telescope_location = telescope_location + self.focal_length = focal_length + self.obswl = observed_wavelength + self.relative_humidity = meteo_params["relative_humidity"] + self.temperature = meteo_params["temperature"] + self.pressure = meteo_params["pressure"] + self.stars = [] + self.visible = [] + self.data = [] + self.errors = [] + # Construct the data here. Stars that were not found are marked in the variable "visible" and use the coordinates (0,0) whenever they can not be seen + for star_list in stars: + self.data.append([]) + self.errors.append([]) + self.visible.append({}) + for star in star_list: + if star.reco_x != np.nan * u.m: + self.visible[-1].update({star.label: True}) + self.data[-1].append(star.reco_x) + self.data[-1].append(star.reco_y) + self.errors[-1].append(star.reco_dx) + self.errors[-1].append(star.reco_dy) + else: + self.visible[-1].update({star.label: False}) + self.data[-1].append(0) + self.data[-1].append(0) + self.errors[-1].append( + 1000.0 + ) # large error value to suppress the stars that were not found + self.errors[-1].append(1000.0) + + for star in stars[0]: + self.stars.append(self.init_star(star.label)) + self.fit_mode = "xy" + self.fit_grid = fit_grid + self.star_motion_model = Model(self.fit_function) + self.fit_summary = None + self.fit_resuts = None + + def init_star(self, star_label): + """ + Initialize StarTracker object for a given star + + :param str star_label: Star label according to NOMAD catalog + + :return: StarTracker object + """ + star = Vizier(catalog="NOMAD").query_constraints(NOMAD1=star_label)[0] + star_coords = SkyCoord( + star["RAJ2000"], + star["DEJ2000"], + unit="deg", + frame="icrs", + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + st = StarTracker( + star_label, + star_coords, + self.telescope_location, + self.focal_length, + self.telescope_pointing, + self.obswl, + self.relative_humidity, + self.temperature, + self.pressure, + ) + return st + + def current_pointing(self, t): + """ + Retrieve current telescope pointing + """ + index = self.times.index(t) + + return self.telescope_pointing[index] + + def fit_function(self, p, t): + """ + Construct the fit function for the pointing correction + + p: Fit parameters + t: Timestamp in UNIX_TAI format + + """ + + time = Time(t, format="unix_tai", scale="utc") + index = self.times.index(time) + coord_list = [] + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + for star in self.stars: + if self.visible[index][star.label]: # if star was visible give the value + x, y = star.position_in_camera_frame(time, new_pointing) + else: # otherwise set it to (0,0) and set a large error value + x, y = (0, 0) + if self.fit_grid == "polar": + x, y = cart2pol(x, y) + coord_list.extend([x]) + coord_list.extend([y]) + + return coord_list + + def fit(self, data, errors, time_range, fit_mode="xy"): + """ + Performs the ODR fit of stars trajectories and saves the results as self.fit_results + + :param array data: Reconstructed star positions, data.shape = (N(stars) * 2, len(time_range)), order: x_1, y_1...x_N, y_N + :param array errors: Uncertainties on the reconstructed star positions. Same shape and order as for the data + :param array time_range: Array of timestamps in UNIX_TAI format + :param array-like(SkyCoord) pointings: Array of telescope pointings in ICRS frame + :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. + """ + self.fit_mode = fit_mode + if self.fit_mode == "radec" or self.fit_mode == "xy": + init_mispointing = [0, 0] + elif self.fit_mode == "y": + init_mispointing = [0] + elif self.fit_mode == "xyz": + init_mispointing = [0, 0, 0] + if errors is not None: + rdata = RealData(x=self.times, y=data, sy=errors) + else: + rdata = RealData(x=self.times, y=data) + odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) + self.fit_summary = odr.run() + if self.fit_mode == "radec": + self.fit_results = pd.DataFrame( + data={ + "dRA": [self.fit_summary.beta[0]], + "dDEC": [self.fit_summary.beta[1]], + "eRA": [self.fit_summary.sd_beta[0]], + "eDEC": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "xy": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "y": + self.fit_results = pd.DataFrame( + data={ + "dY": [self.fit_summary.beta[0]], + "eY": [self.fit_summary.sd_beta[0]], + } + ) + elif self.fit_mode == "xyz": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "dZ": [self.fit_summary.beta[2]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + "eZ": [self.fit_summary.sd_beta[2]], + } + ) From e4bc632bc0f5c6265193cb28841afd141dd5c038 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:31:20 +0200 Subject: [PATCH 098/221] Copying over code for interpolators and pointing calculators --- src/ctapipe/calib/camera/calibrator.py | 206 ++++++++++++- src/ctapipe/image/psf_model.py | 95 ++++++ src/ctapipe/io/interpolation.py | 345 ++++++++++++++++++++++ src/ctapipe/io/tests/test_interpolator.py | 179 +++++++++++ 4 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py create mode 100644 src/ctapipe/io/interpolation.py create mode 100644 src/ctapipe/io/tests/test_interpolator.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 853ba3f7da8..8d9e309c077 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -7,20 +7,34 @@ import astropy.units as u import numpy as np +import Vizier +from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Int, + Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = ["CameraCalibrator"] +__all__ = [ + "CalibrationCalculator", + "TwoPassStatisticsCalculator", + "CameraCalibrator", +] @cache @@ -48,6 +62,196 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + output_path = Path(help="output filename").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_extractor=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class TwoPassStatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) + + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) + + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > self.faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously calculated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + def _get_slice_range( + self, + chunk_nr, + chunk_size, + faultless_previous_chunk, + last_chunk, + last_element, + ): + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + self.chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + self.chunk_shift + ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + + return slice_start, slice_stop + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..458070b8145 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,95 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + + +class PSFModel: + def __init__(self, **kwargs): + """ + Base component to describe image distortion due to the optics of the different cameras. + """ + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py new file mode 100644 index 00000000000..3b792c2107d --- /dev/null +++ b/src/ctapipe/io/interpolation.py @@ -0,0 +1,345 @@ +from abc import ABCMeta, abstractmethod +from typing import Any + +import astropy.units as u +import numpy as np +import tables +from astropy.time import Time +from scipy.interpolate import interp1d + +from ctapipe.core import Component, traits + +from .astropy_helpers import read_table + + +class StepFunction: + + """ + Step function Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the closest previous + point given. + + Parameters + ---------- + values : None | np.array + Numpy array of the data that is to be interpolated. + The first dimension needs to be an index over time + times : None | np.array + Time values over which data are to be interpolated + need to be sorted and have same length as first dimension of values + """ + + def __init__( + self, + times, + values, + bounds_error=True, + fill_value="extrapolate", + assume_sorted=True, + copy=False, + ): + self.values = values + self.times = times + self.bounds_error = bounds_error + self.fill_value = fill_value + + def __call__(self, point): + if point < self.times[0]: + if self.bounds_error: + raise ValueError("below the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[0] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + else: + i = np.searchsorted(self.times, point, side="left") + return self.values[i - 1] + + +class Interpolator(Component, metaclass=ABCMeta): + """ + Interpolator parent class. + + Parameters + ---------- + h5file : None | tables.File + A open hdf5 file with read access. + """ + + bounds_error = traits.Bool( + default_value=True, + help="If true, raises an exception when trying to extrapolate out of the given table", + ).tag(config=True) + + extrapolate = traits.Bool( + help="If bounds_error is False, this flag will specify whether values outside" + "the available values are filled with nan (False) or extrapolated (True).", + default_value=False, + ).tag(config=True) + + telescope_data_group = None + required_columns = set() + expected_units = {} + + def __init__(self, h5file=None, **kwargs): + super().__init__(**kwargs) + + if h5file is not None and not isinstance(h5file, tables.File): + raise TypeError("h5file must be a tables.File") + self.h5file = h5file + + self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) + if self.bounds_error: + self.interp_options["bounds_error"] = True + elif self.extrapolate: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = "extrapolate" + else: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = np.nan + + self._interpolators = {} + + @abstractmethod + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + This method reads input tables and creates instances of the needed interpolators + to be added to _interpolators. The first index of _interpolators needs to be + tel_id, the second needs to be the name of the parameter that is to be interpolated + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are always ``time`` as ``Time`` column and + other columns for the data that is to be interpolated + """ + + pass + + def _check_tables(self, input_table): + missing = self.required_columns - set(input_table.colnames) + if len(missing) > 0: + raise ValueError(f"Table is missing required column(s): {missing}") + for col in self.expected_units: + unit = input_table[col].unit + if unit is None: + if self.expected_units[col] is not None: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + elif not self.expected_units[col].is_equivalent(unit): + if self.expected_units[col] is None: + raise ValueError(f"{col} must have units compatible with 'None'") + else: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + + def _check_interpolators(self, tel_id): + if tel_id not in self._interpolators: + if self.h5file is not None: + self._read_parameter_table(tel_id) # might need to be removed + else: + raise KeyError(f"No table available for tel_id {tel_id}") + + def _read_parameter_table(self, tel_id): + input_table = read_table( + self.h5file, + f"{self.telescope_data_group}/tel_{tel_id:03d}", + ) + self.add_table(tel_id, input_table) + + +class PointingInterpolator(Interpolator): + """ + Interpolator for pointing and pointing correction data + """ + + telescope_data_group = "/dl0/monitoring/telescope/pointing" + required_columns = frozenset(["time", "azimuth", "altitude"]) + expected_units = {"azimuth": u.rad, "altitude": u.rad} + + def __call__(self, tel_id, time): + """ + Interpolate alt/az for given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the pointing + + Returns + ------- + altitude : astropy.units.Quantity[deg] + interpolated altitude angle + azimuth : astropy.units.Quantity[deg] + interpolated azimuth angle + """ + + self._check_interpolators(tel_id) + + mjd = time.tai.mjd + az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) + alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) + return alt, az + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` + as quantity columns for pointing and pointing correction data. + """ + + self._check_tables(input_table) + + if not isinstance(input_table["time"], Time): + raise TypeError("'time' column of pointing table must be astropy.time.Time") + + input_table = input_table.copy() + input_table.sort("time") + + az = input_table["azimuth"].quantity.to_value(u.rad) + # prepare azimuth for interpolation by "unwrapping": i.e. turning + # [359, 1] into [359, 361]. This assumes that if we get values like + # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees + # the other way around. This should be true for all telescopes given + # the sampling speed of pointing values and their maximum movement speed. + # No telescope can turn more than 180° in 2 seconds. + az = np.unwrap(az) + alt = input_table["altitude"].quantity.to_value(u.rad) + mjd = input_table["time"].tai.mjd + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) + self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) + + +class FlatFieldInterpolator(Interpolator): + """ + Interpolator for flatfield data + """ + + telescope_data_group = "dl1/calibration/gain" # TBD + required_columns = frozenset(["time", "gain"]) # TBD + expected_units = {"gain": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate flatfield data for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + ffield : array [float] + interpolated flatfield data + """ + + self._check_interpolators(tel_id) + + ffield = self._interpolators[tel_id]["gain"](time) + return ffield + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "gain" + for the flatfield data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + gain = input_table["gain"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["gain"] = StepFunction( + time, gain, **self.interp_options + ) + + +class PedestalInterpolator(Interpolator): + """ + Interpolator for Pedestal data + """ + + telescope_data_group = "dl1/calibration/pedestal" # TBD + required_columns = frozenset(["time", "pedestal"]) # TBD + expected_units = {"pedestal": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate pedestal or gain for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + pedestal : array [float] + interpolated pedestal values + """ + + self._check_interpolators(tel_id) + + pedestal = self._interpolators[tel_id]["pedestal"](time) + return pedestal + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "pedestal" + for the pedestal data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + pedestal = input_table["pedestal"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["pedestal"] = StepFunction( + time, pedestal, **self.interp_options + ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py new file mode 100644 index 00000000000..930e1e7d73c --- /dev/null +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -0,0 +1,179 @@ +import astropy.units as u +import numpy as np +import pytest +import tables +from astropy.table import Table +from astropy.time import Time + +from ctapipe.io.interpolation import ( + FlatFieldInterpolator, + PedestalInterpolator, + PointingInterpolator, +) + +t0 = Time("2022-01-01T00:00:00") + + +def test_azimuth_switchover(): + """Test pointing interpolation""" + + table = Table( + { + "time": t0 + [0, 1, 2] * u.s, + "azimuth": [359, 1, 3] * u.deg, + "altitude": [60, 61, 62] * u.deg, + }, + ) + + interpolator = PointingInterpolator() + interpolator.add_table(1, table) + + alt, az = interpolator(tel_id=1, time=t0 + 0.5 * u.s) + assert u.isclose(az, 360 * u.deg) + assert u.isclose(alt, 60.5 * u.deg) + + +def test_invalid_input(): + """Test invalid pointing tables raise nice errors""" + + wrong_time = Table( + { + "time": [1, 2, 3] * u.s, + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3] * u.deg, + } + ) + + interpolator = PointingInterpolator() + with pytest.raises(TypeError, match="astropy.time.Time"): + interpolator.add_table(1, wrong_time) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.m, + "altitude": [1, 2, 3] * u.deg, + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3], + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + +def test_hdf5(tmp_path): + """Test writing interpolated data to file""" + from ctapipe.io import write_table + + table = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + path = tmp_path / "pointing.h5" + write_table(table, path, "/dl0/monitoring/telescope/pointing/tel_001") + with tables.open_file(path) as h5file: + interpolator = PointingInterpolator(h5file) + alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + +def test_bounds(): + """Test invalid pointing tables raise nice errors""" + + table_pointing = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + table_pedestal = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + table_flatfield = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + interpolator_pointing = PointingInterpolator() + interpolator_pedestal = PedestalInterpolator() + interpolator_flatfield = FlatFieldInterpolator() + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + error_message = "below the interpolation range" + + with pytest.raises(ValueError, match=error_message): + interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) + + with pytest.raises(ValueError, match=error_message): + interpolator_pedestal(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match=error_message): + interpolator_flatfield(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match="above the interpolation range"): + interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + pedestal = interpolator_pedestal(tel_id=1, time=1.0) + assert all(pedestal == table_pedestal["pedestal"][0]) + flatfield = interpolator_flatfield(tel_id=1, time=1.0) + assert all(flatfield == table_flatfield["gain"][0]) + with pytest.raises(KeyError): + interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) + with pytest.raises(KeyError): + interpolator_pedestal(tel_id=2, time=1.0) + with pytest.raises(KeyError): + interpolator_flatfield(tel_id=2, time=1.0) + + interpolator_pointing = PointingInterpolator(bounds_error=False) + interpolator_pedestal = PedestalInterpolator(bounds_error=False) + interpolator_flatfield = FlatFieldInterpolator(bounds_error=False) + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + for dt in (-0.1, 10.1) * u.s: + alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) + assert np.isnan(alt.value) + assert np.isnan(az.value) + + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) + interpolator_pointing.add_table(1, table_pointing) + alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) + assert u.isclose(alt, 71 * u.deg) + assert u.isclose(az, -1 * u.deg) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 11 * u.s) + assert u.isclose(alt, 59 * u.deg) + assert u.isclose(az, 11 * u.deg) From 3b147c30584098b99e999e942e9f94c2688dffab Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:50:51 +0200 Subject: [PATCH 099/221] Fixed some issues with the ChunkFunction --- src/ctapipe/calib/camera/calibrator.py | 3 +- src/ctapipe/io/interpolation.py | 62 ++++++++++++++++------- src/ctapipe/io/tests/test_interpolator.py | 9 +++- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 8d9e309c077..bcc05ee3c71 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -7,7 +7,7 @@ import astropy.units as u import numpy as np -import Vizier +import Vizier # discuss this dependency with max etc. from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 @@ -251,7 +251,6 @@ def _get_slice_range( return slice_start, slice_stop - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py index 3b792c2107d..e0e27470c99 100644 --- a/src/ctapipe/io/interpolation.py +++ b/src/ctapipe/io/interpolation.py @@ -12,12 +12,13 @@ from .astropy_helpers import read_table -class StepFunction: +class ChunkFunction: """ - Step function Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the closest previous - point given. + Chunk Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the latest starting + valid chunk is given or the earliest available still valid chunk for any + pixels without valid data. Parameters ---------- @@ -31,7 +32,8 @@ class StepFunction: def __init__( self, - times, + start_times, + end_times, values, bounds_error=True, fill_value="extrapolate", @@ -39,12 +41,13 @@ def __init__( copy=False, ): self.values = values - self.times = times + self.start_times = start_times + self.end_times = end_times self.bounds_error = bounds_error self.fill_value = fill_value def __call__(self, point): - if point < self.times[0]: + if point < self.start_times[0]: if self.bounds_error: raise ValueError("below the interpolation range") @@ -56,9 +59,28 @@ def __call__(self, point): a[:] = np.nan return a + elif point > self.end_times[-1]: + if self.bounds_error: + raise ValueError("above the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[-1] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + else: - i = np.searchsorted(self.times, point, side="left") - return self.values[i - 1] + i = np.searchsorted( + self.start_times, point, side="left" + ) # Latest valid chunk + j = np.searchsorted( + self.end_times, point, side="left" + ) # Earliest valid chunk + return np.where( + np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] + ) # Give value for latest chunk unless its nan. If nan give earliest chunk value class Interpolator(Component, metaclass=ABCMeta): @@ -237,7 +259,7 @@ class FlatFieldInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["time", "gain"]) # TBD + required_columns = frozenset(["start_time", "end_time", "gain"]) expected_units = {"gain": u.one} def __call__(self, tel_id, time): @@ -279,12 +301,13 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] gain = input_table["gain"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = StepFunction( - time, gain, **self.interp_options + self._interpolators[tel_id]["gain"] = ChunkFunction( + start_time, end_time, gain, **self.interp_options ) @@ -294,7 +317,7 @@ class PedestalInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["time", "pedestal"]) # TBD + required_columns = frozenset(["start_time", "end_time", "pedestal"]) expected_units = {"pedestal": u.one} def __call__(self, tel_id, time): @@ -336,10 +359,11 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] pedestal = input_table["pedestal"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = StepFunction( - time, pedestal, **self.interp_options + self._interpolators[tel_id]["pedestal"] = ChunkFunction( + start_time, end_time, pedestal, **self.interp_options ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py index 930e1e7d73c..20f5657c1ae 100644 --- a/src/ctapipe/io/tests/test_interpolator.py +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -103,7 +103,8 @@ def test_bounds(): table_pedestal = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -111,7 +112,8 @@ def test_bounds(): table_flatfield = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -168,6 +170,9 @@ def test_bounds(): assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=20.0))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=20.0))) + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) interpolator_pointing.add_table(1, table_pointing) alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) From 4bd5cb3c882324c6a8b7dfa657df58a483e9e3aa Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:53:05 +0200 Subject: [PATCH 100/221] Adding the StatisticsExtractors --- src/ctapipe/calib/camera/extractor.py | 233 ++++++++++++++++++ .../calib/camera/tests/test_extractors.py | 84 +++++++ 2 files changed, 317 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..7093d057f20 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,233 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", + "SigmaClippingExtractor", +] + +from abc import abstractmethod + +import numpy as np +from astropy.stats import sigma_clipped_stats + +from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + Int, + List, +) + + +class StatisticsExtractor(TelescopeComponent): + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape + (n_images, n_channels, n_pix). + col_name : string + column name in the dl1 table + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + # in python 3.12 itertools.batched can be used + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) + return stats_list + + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + # ensure numpy array + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + # median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + ) + + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: + # in python 3.12 itertools.batched can be used + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + # ensure numpy array + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + + # median of the event images + # image_median = np.ma.median(masked_images, axis=-1) + + # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + masked_images, + sigma=max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + # outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), + std_outliers=image_std_outliers.filled(True), + ) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..a83c93fd1c0 --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,84 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import numpy as np +import pytest +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + + +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): + """test basic functionality of the StatisticsExtractors""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) + + +def test_check_outliers(test_sigmaclippingextractor): + """test detection ability of outliers""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 From 509e51274defad90939c1a14828e47ceaf3c2f64 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 3 Sep 2024 15:53:10 +0200 Subject: [PATCH 101/221] I added a basic star fitter --- src/ctapipe/calib/camera/calibrator.py | 209 +-------- src/ctapipe/calib/camera/pointing.py | 571 +++++++++++++++++++++++++ src/ctapipe/containers.py | 29 ++ 3 files changed, 604 insertions(+), 205 deletions(-) create mode 100644 src/ctapipe/calib/camera/pointing.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index bcc05ee3c71..6ad47b525fe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,40 +2,29 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +<<<<<<< HEAD +======= +>>>>>>> c5f385f0 (I added a basic star fitter) from functools import cache import astropy.units as u import numpy as np -import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, - Float, - Int, - Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "CameraCalibrator", -] +__all__ = ["CameraCalibrator"] @cache def _get_pixel_index(n_pixels): @@ -61,196 +50,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels - -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py new file mode 100644 index 00000000000..2f89159bcdf --- /dev/null +++ b/src/ctapipe/calib/camera/pointing.py @@ -0,0 +1,571 @@ +""" +Definition of the `CameraCalibrator` class, providing all steps needed to apply +calibration and image extraction, as well as supporting algorithms. +""" + +import copy +from functools import cache + +import astropy.units as u +import numpy as np +import Vizier # discuss this dependency with max etc. +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import StatisticsExtractor +from ctapipe.containers import StarContainer +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + ComponentName, + Dict, + Float, + Integer, + TelescopeParameter, +) +from ctapipe.image import tailcuts_clean +from ctapipe.image.psf_model import PSFModel +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader + +__all__ = [ + "PointingCalculator", +] + + +@cache +def _get_pixel_index(n_pixels): + """Cached version of ``np.arange(n_pixels)``""" + return np.arange(n_pixels) + + +def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channel): + broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) + + index = _get_pixel_index(n_pixels) + masks = ( + pixel_status.hardware_failing_pixels, + pixel_status.pedestal_failing_pixels, + pixel_status.flatfield_failing_pixels, + ) + for mask in masks: + if mask is not None: + if selected_gain_channel is not None: + broken_pixels |= mask[selected_gain_channel, index] + else: + broken_pixels |= mask + + return broken_pixels + + +def cart2pol(x, y): + """ + Convert cartesian coordinates to polar + + :param float x: X coordinate [m] + :param float y: Y coordinate [m] + + :return: Tuple (r, φ)[m, rad] + """ + rho = np.sqrt(x**2 + y**2) + phi = np.arctan2(y, x) + return (rho, phi) + + +def pol2cart(rho, phi): + """ + Convert polar coordinates to cartesian + + :param float rho: R coordinate + :param float phi: ¢ coordinate [rad] + + :return: Tuple (x,y)[m, m] + """ + x = rho * np.cos(phi) + y = rho * np.sin(phi) + return (x, y) + + +class PointingCalculator(TelescopeComponent): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + cleaning = Dict( + {"bound_thresh": 750, "pic_thresh": 15000}, help="Image cleaning parameters" + ).tag(config=True) + + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + + psf_model_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="ComaModel", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__( + subarray=subarray, + stats_extractor="Plain", + config=config, + parent=parent, + **kwargs, + ) + + self.psf = PSFModel.from_name( + self.psf_model_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, input_url, tel_id): + self.tel_id = tel_id + + if self._check_req_data(input_url, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + # first get the camera geometry and pointing for the file and determine what stars we should see + + with EventSource(input_url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) # get some pointing to make a list of stars that we expect to see + + self.pointing = self.pointing.transform_to("icrs") + + self.broken_pixels = np.unique(np.where(self.broken_pixels)) + + self.image_size = len( + event.variance_image.image + ) # get the size of images of the camera we are calibrating + + self.stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] # get all stars that could be in the fov + + self.stars_in_fov = self.stars_in_fov[ + self.tars_in_fov["Bmag"] < self.max_star_magnitude + ] # select stars for magnitude to exclude those we would not be able to see + + # get the accumulated variance images + + ( + accumulated_pointing, + accumulated_times, + variance_statistics, + ) = self._get_accumulated_images(input_url) + + accumulated_images = np.array([x.mean for x in variance_statistics]) + + star_pixels = self._get_expected_star_pixels( + accumulated_times, accumulated_pointing + ) + + star_mask = np.ones(self.image_size, dtype=bool) + + star_mask[star_pixels] = False + + # get NSB values + + nsb = np.mean(accumulated_images[star_mask], axis=1) + nsb_std = np.std(accumulated_images[star_mask], axis=1) + + clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) + + reco_stars = [] + + for i, image in enumerate(clean_images): + reco_stars.append([]) + camera_frame = EngineeringCameraFrame( + telescope_pointing=accumulated_pointing[i], + focal_length=self.focal_length, + obstime=accumulated_times[i].utc, + location=self.location, + ) + for star in self.stars_in_fov: + reco_stars[-1].append( + self._fit_star_position( + star, accumulated_times[i], camera_frame, image, nsb_std[i] + ) + ) + + return reco_stars + + # now fit the star locations + + def _check_req_data(self, url, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[self.tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + + def _calibrate_var_images(self, var_images, time, calibration_file): + """ + Calibrate a set of variance images + + Parameters + ---------- + var_images : list + list of variance images + time : list + list of times correxponding to the variance images + calibration_file : str + name of the file where the calibration data can be found + """ + # So i need to use the interpolator classes to read the calibration data + relative_gains = FlatFieldInterpolator( + calibration_file + ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, + np.square(relative_gains(time[i])), + ) + + return var_images + + def _get_expected_star_pixels(self, time_list, pointing_list): + """ + Determine which in which pixels stars are expected for a series of images + + Parameters + ---------- + time_list : list + list of time values where the images were capturedd + pointing_list : list + list of pointing values for the images + """ + + res = [] + + for pointing, time in zip( + pointing_list, time_list + ): # loop over time and pointing of images + temp = [] + + camera_frame = EngineeringCameraFrame( + telescope_pointing=pointing, + focal_length=self.focal_length, + obstime=time.utc, + location=self.location, + ) # get the engineering camera frame for the pointing + + for star in self.stars_in_fov: + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + expected_central_pixel = self.camera_geometry.transform_to( + camera_frame + ).position_to_pix_index( + star_coords.x, star_coords.y + ) # get where the star should be + cluster = copy.deepcopy( + self.camera_geometry.neighbors[expected_central_pixel] + ) # get the neighborhood of the star + cluster_corona = [] + + for pixel_index in cluster: + cluster_corona.extend( + copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) + ) # and add another layer of pixels to be sure + + cluster.extend(cluster_corona) + cluster.append(expected_central_pixel) + temp.extend(list(set(cluster))) + + res.append(temp) + + return res + + def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + + rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) + + if phi < 0: + phi = phi + 2 * np.pi + + star_container = StarContainer( + label=star["NOMAD1"], + magnitude=star["Bmag"], + expected_x=star_coords.x, + expected_y=star_coords.y, + expected_r=rho * u.m, + expected_phi=phi * u.rad, + timestamp=timestamp, + ) + + current_geometry = self.camera_geometry.transform_to(camera_frame) + + hit_pdf = self._get_star_pdf(star, current_geometry) + cluster = np.where(hit_pdf > self.pdf_percentile_limit * np.sum(hit_pdf)) + + if not np.any(image[cluster] > self.min_star_prominence * nsb_std): + self.log.info("Star %s can not be detected", star["NOMAD1"]) + star.pixels = np.full(self.max_cluster_size, -1) + return star_container + + pad_size = self.max_cluster_size - len(cluster[0]) + if pad_size > 0: + star.pixels = np.pad(cluster[0], (0, pad_size), constant_values=-1) + else: + star.pixels = cluster[0][: self.max_cluster_size] + + self.log.warning( + "Reconstructed cluster is longer than %s, truncated cluster info will " + "be recorded to the output table. Not a big deal, as correct cluster " + "used for position reconstruction.", + self.max_cluster_size, + ) + return star_container + + rs, fs = cart2pol( + current_geometry.pix_x[cluster].to_value(u.m), + current_geometry.pix_y[cluster].to_value(u.m), + ) + + k, r0, sr = self.psf_model.radial_pdf_params + + star_container.reco_r = ( + self.coma_r_shift_correction + * np.average(rs, axis=None, weights=image[cluster], returned=False) + * u.m + ) + + star_container.reco_x = self.coma_r_shift_correction * np.average( + current_geometry.pix_x[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + star_container.reco_y = self.coma_r_shift_correction * np.average( + current_geometry.pix_y[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + _, star_container.reco_phi = cart2pol(star.reco_x, star.reco_y) + + if star_container.reco_phi < 0: + star_container.reco_phi = star.reco_phi + 2 * np.pi * u.rad + + star_container.reco_dx = ( + np.sqrt(np.cov(current_geometry.pix_x[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dy = ( + np.sqrt(np.cov(current_geometry.pix_y[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dr = np.sqrt(np.cov(rs, aweights=hit_pdf[cluster])) * u.m + + _, star_container.reco_dphi = cart2pol( + star_container.reco_dx, star_container.reco_dy + ) + + return star_container + + def _get_accumulated_images(self, input_url): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=self.tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=True, + ) + + # get the trigger type for all images and make a mask + + event_mask = dl1_table["event_type"] == 2 + + # get the pointing for all images and filter for trigger type + + altitude = dl1_table["telescope_pointing_altitude"][event_mask] + azimuth = dl1_table["telescope_pointing_azimuth"][event_mask] + time = dl1_table["time"][event_mask] + + pointing = [ + SkyCoord( + az=x, alt=y, frame="altaz", obstime=z.tai.utc, location=self.location + ) + for x, y, z in zip(azimuth, altitude, time) + ] + + # get the time and images from the data + + variance_images = copy.deepcopy(dl1_table["variance_image"][event_mask]) + + # now make a filter to to reject EAS light and starlight and keep a separate EAS filter + + charge_images = dl1_table["image"][event_mask] + + light_mask = [ + tailcuts_clean( + self.camera_geometry, + x, + picture_thresh=self.cleaning["pic_thresh"], + boundary_thresh=self.cleaning["bound_thresh"], + ) + for x in charge_images + ] + + shower_mask = copy.deepcopy(light_mask) + + star_pixels = self._get_expected_star_pixels(time, pointing) + + light_mask[:, star_pixels] = True + + if self.broken_pixels is not None: + light_mask[:, self.broken_pixels] = True + + # calculate the average variance in viable pixels and replace the values where there is EAS light + + mean_variance = np.mean(variance_images[~light_mask]) + + variance_images[shower_mask] = mean_variance + + # now calibrate the images + + variance_images = self._calibrate_var_images( + self, variance_images, time, input_url + ) + + # Get the average variance across the data to + + # then turn it into a table that the extractor can read + variance_image_table = QTable([time, variance_images], names=["time", "image"]) + + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[self.tel_id]] + + # get the cumulative variance images using the statistics extractor and return the value + + variance_statistics = extractor(variance_image_table) + + accumulated_times = np.array([x.validity_start for x in variance_statistics]) + + # calculate where stars might be + + accumulated_pointing = np.array( + [x for x in pointing if pointing.time in accumulated_times] + ) + + return (accumulated_pointing, accumulated_times, variance_statistics) + + def _get_star_pdf(self, star, current_geometry): + image = np.zeros(self.image_size) + + r0 = star.expected_r.to_value(u.m) + f0 = star.expected_phi.to_value(u.rad) + + self.psf_model.update_model_parameters(r0, f0) + + dr = ( + self.pdf_bin_size + * np.rad2deg(np.arctan(1 / self.focal_length.to_value(u.m))) + / 3600.0 + ) + r = np.linspace( + r0 - dr * self.n_pdf_bins / 2.0, + r0 + dr * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + df = np.deg2rad(self.pdf_bin_size / 3600.0) * 100 + f = np.linspace( + f0 - df * self.n_pdf_bins / 2.0, + f0 + df * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + + for r_ in r: + for f_ in f: + val = self.psf_model.pdf(r_, f_) * dr * df + x, y = pol2cart(r_, f_) + pixelN = current_geometry.position_to_pix_index(x * u.m, y * u.m) + if pixelN != -1: + image[pixelN] += val + + return image diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 311142311a3..489042a1953 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -64,6 +64,7 @@ "ObservationBlockContainer", "ObservingMode", "ObservationBlockState", + "StarContainer", ] @@ -1529,3 +1530,31 @@ class ObservationBlockContainer(Container): scheduled_start_time = Field(NAN_TIME, "expected start time from scheduler") actual_start_time = Field(NAN_TIME, "true start time") actual_duration = Field(nan * u.min, "true duration", unit=u.min) + + +class StarContainer(Container): + "Stores information about a star in the field of view of a camera." + + label = Field("", "Star label", dtype=np.str_) + magnitude = Field(-1, "Star magnitude") + expected_x = Field(np.nan * u.m, "Expected star position (x)", unit=u.m) + expected_y = Field(np.nan * u.m, "Expected star position (y)", unit=u.m) + + expected_r = Field(np.nan * u.m, "Expected star position (r)", unit=u.m) + expected_phi = Field(np.nan * u.rad, "Expected star position (phi)", unit=u.rad) + + reco_x = Field(np.nan * u.m, "Reconstructed star position (x)", unit=u.m) + reco_y = Field(np.nan * u.m, "Reconstructed star position (y)", unit=u.m) + reco_dx = Field(np.nan * u.m, "Reconstructed star position error (x)", unit=u.m) + reco_dy = Field(np.nan * u.m, "Reconstructed star position error (y)", unit=u.m) + + reco_r = Field(np.nan * u.m, "Reconstructed star position (r)", unit=u.m) + reco_phi = Field(np.nan * u.rad, "Reconstructed star position (phi)", unit=u.rad) + reco_dr = Field(np.nan * u.m, "Reconstructed star position error (r)", unit=u.m) + reco_dphi = Field( + np.nan * u.rad, "Reconstructed star position error (phi)", unit=u.rad + ) + + timestamp = Field(NAN_TIME, "Reconstruction timestamp") + + pixels = Field(np.full(20, -1), "List of star pixel ids", dtype=np.int_, ndim=1) From 69eb08e737eeefc05ecd3e862f17a8d40e2a2f24 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 12 Sep 2024 11:08:20 +0200 Subject: [PATCH 102/221] Added the fitting --- src/ctapipe/calib/camera/pointing.py | 326 ++++++++++++++++++++++++++- 1 file changed, 322 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 2f89159bcdf..77105194382 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -8,9 +8,12 @@ import astropy.units as u import numpy as np +import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord from astropy.table import QTable +from astropy.time import Time +from scipy.odr import ODR, Model, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer @@ -85,6 +88,87 @@ def pol2cart(rho, phi): return (x, y) +class StarTracker: + """ + Utility class to provide the position of the star in the telescope's camera frame coordinates at a given time + """ + + def __init__( + self, + star_label, + star_coordinates, + telescope_location, + telescope_focal_length, + telescope_pointing, + observed_wavelength, + relative_humidity, + temperature, + pressure, + pointing_label=None, + ): + """ + Constructor + + :param str star_label: Star label + :param SkyCoord star_coordinates: Star coordinates in ICRS frame + :param EarthLocation telescope_location: Telescope location coordinates + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str pointing_label: Pointing label + """ + self.star_label = star_label + self.star_coordinates_icrs = star_coordinates + self.telescope_location = telescope_location + self.telescope_focal_length = telescope_focal_length + self.telescope_pointing = telescope_pointing + self.obswl = observed_wavelength + self.relative_humidity = relative_humidity + self.temperature = temperature + self.pressure = pressure + self.pointing_label = pointing_label + + def position_in_camera_frame(self, timestamp, pointing=None, focal_correction=0): + """ + Calculates star position in the engineering camera frame + + :param astropy.Time timestamp: Timestamp of the observation + :param SkyCoord pointing: Current telescope pointing in ICRS frame + :param float focal_correction: Correction to the focal length of the telescope. Float, should be provided in meters + + :return: Pair (float, float) of star's (x,y) coordinates in the engineering camera frame in meters + """ + # If no telescope pointing is provided, use the telescope pointing, provided + # during the class member initialization + if pointing is None: + pointing = self.telescope_pointing + # Determine current telescope pointing in AltAz + altaz_pointing = pointing.transform_to( + AltAz( + obstime=timestamp, + location=self.telescope_location, + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + ) + # Create current camera frame + camera_frame = EngineeringCameraFrame( + telescope_pointing=altaz_pointing, + focal_length=self.telescope_focal_length + focal_correction * u.m, + obstime=timestamp, + location=self.telescope_location, + ) + # Calculate the star's coordinates in the current camera frame + star_coords_camera = self.star_coordinates_icrs.transform_to(camera_frame) + return (star_coords_camera.x.to_value(), star_coords_camera.y.to_value()) + + class PointingCalculator(TelescopeComponent): """ Component to calculate pointing corrections from interleaved skyfield events. @@ -103,6 +187,12 @@ class PointingCalculator(TelescopeComponent): "elevation - in meters", ).tag(config=True) + observed_wavelength = Float( + 0.35, + help="Observed star light wavelength in microns" + "(convolution of blackbody spectrum with camera sensitivity)", + ).tag(config=True) + min_star_prominence = Integer( 3, help="Minimal star prominence over the background in terms of " @@ -128,6 +218,11 @@ class PointingCalculator(TelescopeComponent): help="Name of the PSFModel Subclass to be used.", ).tag(config=True) + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + def __init__( self, subarray, @@ -196,6 +291,8 @@ def __call__(self, input_url, tel_id): self.tars_in_fov["Bmag"] < self.max_star_magnitude ] # select stars for magnitude to exclude those we would not be able to see + star_labels = [x.label for x in self.stars_in_fov] + # get the accumulated variance images ( @@ -240,7 +337,17 @@ def __call__(self, input_url, tel_id): return reco_stars - # now fit the star locations + # now fit the pointing correction. + fitter = Pointing_Fitter( + star_labels, + self.pointing, + self.location, + self.focal_length, + self.observed_wavelength, + self.meteo_parameters, + ) + + fitter.fit(accumulated_pointing) def _check_req_data(self, url, calibration_type): """ @@ -299,7 +406,7 @@ def _get_expected_star_pixels(self, time_list, pointing_list): Parameters ---------- time_list : list - list of time values where the images were capturedd + list of time values where the images were captured pointing_list : list list of pointing values for the images """ @@ -346,10 +453,13 @@ def _get_expected_star_pixels(self, time_list, pointing_list): return res - def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + def _fit_star_position( + self, star, timestamp, camera_frame, image, nsb_std, current_pointing + ): star_coords = SkyCoord( star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" ) + star_coords = star_coords.transform_to(camera_frame) rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) @@ -569,3 +679,211 @@ def _get_star_pdf(self, star, current_geometry): image[pixelN] += val return image + + +class Pointing_Fitter: + """ + Pointing correction fitter + """ + + def __init__( + self, + stars, + times, + telescope_pointing, + telescope_location, + focal_length, + observed_wavelength, + meteo_params, + fit_grid="polar", + ): + """ + Constructor + + :param list stars: List of lists of star containers the first dimension is supposed to be time + :param list time: List of time values for the when the star locations were fitted + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param EarthLocation telescope_location: Telescope location + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str fit_grid: Coordinate system grid to use. Either polar or cartesian + """ + self.star_containers = stars + self.times = times + self.telescope_pointing = telescope_pointing + self.telescope_location = telescope_location + self.focal_length = focal_length + self.obswl = observed_wavelength + self.relative_humidity = meteo_params["relative_humidity"] + self.temperature = meteo_params["temperature"] + self.pressure = meteo_params["pressure"] + self.stars = [] + self.visible = [] + self.data = [] + self.errors = [] + # Construct the data here. Stars that were not found are marked in the variable "visible" and use the coordinates (0,0) whenever they can not be seen + for star_list in stars: + self.data.append([]) + self.errors.append([]) + self.visible.append({}) + for star in star_list: + if star.reco_x != np.nan * u.m: + self.visible[-1].update({star.label: True}) + self.data[-1].append(star.reco_x) + self.data[-1].append(star.reco_y) + self.errors[-1].append(star.reco_dx) + self.errors[-1].append(star.reco_dy) + else: + self.visible[-1].update({star.label: False}) + self.data[-1].append(0) + self.data[-1].append(0) + self.errors[-1].append( + 1000.0 + ) # large error value to suppress the stars that were not found + self.errors[-1].append(1000.0) + + for star in stars[0]: + self.stars.append(self.init_star(star.label)) + self.fit_mode = "xy" + self.fit_grid = fit_grid + self.star_motion_model = Model(self.fit_function) + self.fit_summary = None + self.fit_resuts = None + + def init_star(self, star_label): + """ + Initialize StarTracker object for a given star + + :param str star_label: Star label according to NOMAD catalog + + :return: StarTracker object + """ + star = Vizier(catalog="NOMAD").query_constraints(NOMAD1=star_label)[0] + star_coords = SkyCoord( + star["RAJ2000"], + star["DEJ2000"], + unit="deg", + frame="icrs", + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + st = StarTracker( + star_label, + star_coords, + self.telescope_location, + self.focal_length, + self.telescope_pointing, + self.obswl, + self.relative_humidity, + self.temperature, + self.pressure, + ) + return st + + def current_pointing(self, t): + """ + Retrieve current telescope pointing + """ + index = self.times.index(t) + + return self.telescope_pointing[index] + + def fit_function(self, p, t): + """ + Construct the fit function for the pointing correction + + p: Fit parameters + t: Timestamp in UNIX_TAI format + + """ + + time = Time(t, format="unix_tai", scale="utc") + index = self.times.index(time) + coord_list = [] + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + for star in self.stars: + if self.visible[index][star.label]: # if star was visible give the value + x, y = star.position_in_camera_frame(time, new_pointing) + else: # otherwise set it to (0,0) and set a large error value + x, y = (0, 0) + if self.fit_grid == "polar": + x, y = cart2pol(x, y) + coord_list.extend([x]) + coord_list.extend([y]) + + return coord_list + + def fit(self, data, errors, time_range, fit_mode="xy"): + """ + Performs the ODR fit of stars trajectories and saves the results as self.fit_results + + :param array data: Reconstructed star positions, data.shape = (N(stars) * 2, len(time_range)), order: x_1, y_1...x_N, y_N + :param array errors: Uncertainties on the reconstructed star positions. Same shape and order as for the data + :param array time_range: Array of timestamps in UNIX_TAI format + :param array-like(SkyCoord) pointings: Array of telescope pointings in ICRS frame + :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. + """ + self.fit_mode = fit_mode + if self.fit_mode == "radec" or self.fit_mode == "xy": + init_mispointing = [0, 0] + elif self.fit_mode == "y": + init_mispointing = [0] + elif self.fit_mode == "xyz": + init_mispointing = [0, 0, 0] + if errors is not None: + rdata = RealData(x=self.times, y=data, sy=errors) + else: + rdata = RealData(x=self.times, y=data) + odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) + self.fit_summary = odr.run() + if self.fit_mode == "radec": + self.fit_results = pd.DataFrame( + data={ + "dRA": [self.fit_summary.beta[0]], + "dDEC": [self.fit_summary.beta[1]], + "eRA": [self.fit_summary.sd_beta[0]], + "eDEC": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "xy": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "y": + self.fit_results = pd.DataFrame( + data={ + "dY": [self.fit_summary.beta[0]], + "eY": [self.fit_summary.sd_beta[0]], + } + ) + elif self.fit_mode == "xyz": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "dZ": [self.fit_summary.beta[2]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + "eZ": [self.fit_summary.sd_beta[2]], + } + ) From da561d2a97bd45cc3415ba8a2ee81bb59e38aae6 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 20 Sep 2024 14:31:30 +0200 Subject: [PATCH 103/221] Finihed V1 of the startracker code --- src/ctapipe/calib/camera/pointing.py | 90 +++++++++++----------------- 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 77105194382..c530eacdfb4 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -291,8 +291,6 @@ def __call__(self, input_url, tel_id): self.tars_in_fov["Bmag"] < self.max_star_magnitude ] # select stars for magnitude to exclude those we would not be able to see - star_labels = [x.label for x in self.stars_in_fov] - # get the accumulated variance images ( @@ -335,19 +333,22 @@ def __call__(self, input_url, tel_id): ) ) - return reco_stars + # now fit the pointing corrections. + correction = [] + for i, stars in enumerate(reco_stars): + fitter = Pointing_Fitter( + stars, + accumulated_times[i], + accumulated_pointing[i], + self.location, + self.focal_length, + self.observed_wavelength, + self.meteo_parameters, + ) - # now fit the pointing correction. - fitter = Pointing_Fitter( - star_labels, - self.pointing, - self.location, - self.focal_length, - self.observed_wavelength, - self.meteo_parameters, - ) + correction.append(fitter.fit()) - fitter.fit(accumulated_pointing) + return correction def _check_req_data(self, url, calibration_type): """ @@ -689,7 +690,7 @@ class Pointing_Fitter: def __init__( self, stars, - times, + time, telescope_pointing, telescope_location, focal_length, @@ -712,7 +713,7 @@ def __init__( :param str fit_grid: Coordinate system grid to use. Either polar or cartesian """ self.star_containers = stars - self.times = times + self.time = time self.telescope_pointing = telescope_pointing self.telescope_location = telescope_location self.focal_length = focal_length @@ -721,32 +722,16 @@ def __init__( self.temperature = meteo_params["temperature"] self.pressure = meteo_params["pressure"] self.stars = [] - self.visible = [] self.data = [] self.errors = [] - # Construct the data here. Stars that were not found are marked in the variable "visible" and use the coordinates (0,0) whenever they can not be seen - for star_list in stars: - self.data.append([]) - self.errors.append([]) - self.visible.append({}) - for star in star_list: - if star.reco_x != np.nan * u.m: - self.visible[-1].update({star.label: True}) - self.data[-1].append(star.reco_x) - self.data[-1].append(star.reco_y) - self.errors[-1].append(star.reco_dx) - self.errors[-1].append(star.reco_dy) - else: - self.visible[-1].update({star.label: False}) - self.data[-1].append(0) - self.data[-1].append(0) - self.errors[-1].append( - 1000.0 - ) # large error value to suppress the stars that were not found - self.errors[-1].append(1000.0) - - for star in stars[0]: - self.stars.append(self.init_star(star.label)) + # Construct the data here. Add only stars that were found + for star in stars: + if not np.isnan(star.reco_x): + self.data.append(star.reco_x) + self.data.append(star.reco_y) + self.errors.append(star.reco_dx) + self.errors.append(star.reco_dy) + self.stars.append(self.init_star(star.label)) self.fit_mode = "xy" self.fit_grid = fit_grid self.star_motion_model = Model(self.fit_function) @@ -803,24 +788,20 @@ def fit_function(self, p, t): """ time = Time(t, format="unix_tai", scale="utc") - index = self.times.index(time) coord_list = [] m_ra, m_dec = p - new_ra = self.current_pointing(time).ra + m_ra * u.deg - new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_ra = self.current_pointing(t).ra + m_ra * u.deg + new_dec = self.current_pointing(t).dec + m_dec * u.deg new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) m_ra, m_dec = p - new_ra = self.current_pointing(time).ra + m_ra * u.deg - new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_ra = self.current_pointing(t).ra + m_ra * u.deg + new_dec = self.current_pointing(t).dec + m_dec * u.deg new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) for star in self.stars: - if self.visible[index][star.label]: # if star was visible give the value - x, y = star.position_in_camera_frame(time, new_pointing) - else: # otherwise set it to (0,0) and set a large error value - x, y = (0, 0) + x, y = star.position_in_camera_frame(time, new_pointing) if self.fit_grid == "polar": x, y = cart2pol(x, y) coord_list.extend([x]) @@ -828,14 +809,10 @@ def fit_function(self, p, t): return coord_list - def fit(self, data, errors, time_range, fit_mode="xy"): + def fit(self, fit_mode="xy"): """ Performs the ODR fit of stars trajectories and saves the results as self.fit_results - :param array data: Reconstructed star positions, data.shape = (N(stars) * 2, len(time_range)), order: x_1, y_1...x_N, y_N - :param array errors: Uncertainties on the reconstructed star positions. Same shape and order as for the data - :param array time_range: Array of timestamps in UNIX_TAI format - :param array-like(SkyCoord) pointings: Array of telescope pointings in ICRS frame :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. """ self.fit_mode = fit_mode @@ -845,10 +822,10 @@ def fit(self, data, errors, time_range, fit_mode="xy"): init_mispointing = [0] elif self.fit_mode == "xyz": init_mispointing = [0, 0, 0] - if errors is not None: - rdata = RealData(x=self.times, y=data, sy=errors) + if self.errors is not None: + rdata = RealData(x=self.time, y=self.data, sy=self.errors) else: - rdata = RealData(x=self.times, y=data) + rdata = RealData(x=self.time, y=self.data) odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) self.fit_summary = odr.run() if self.fit_mode == "radec": @@ -887,3 +864,4 @@ def fit(self, data, errors, time_range, fit_mode="xy"): "eZ": [self.fit_summary.sd_beta[2]], } ) + return self.fit_results From 32894fd81dd152f862b30ce324cf4d7ee8c21e8f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 14:45:27 +0200 Subject: [PATCH 104/221] I separated the star tracking out to the StarTracer class --- src/ctapipe/calib/camera/pointing.py | 918 ++++++++++++--------------- src/ctapipe/image/psf_model.py | 50 +- 2 files changed, 436 insertions(+), 532 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index c530eacdfb4..933105efeb6 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -10,10 +10,9 @@ import numpy as np import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord +from astropy.coordinates import Angle, EarthLocation, SkyCoord from astropy.table import QTable -from astropy.time import Time -from scipy.odr import ODR, Model, RealData +from scipy.odr import ODR, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer @@ -28,11 +27,10 @@ ) from ctapipe.image import tailcuts_clean from ctapipe.image.psf_model import PSFModel -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader +from ctapipe.instrument import CameraGeometry +from ctapipe.io import FlatFieldInterpolator, PointingInterpolator -__all__ = [ - "PointingCalculator", -] +__all__ = ["PointingCalculator", "StarImageGenerator"] @cache @@ -60,6 +58,19 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +def get_index_step(val, lookup): + index = 0 + + for i, x in enumerate(lookup): + if val <= x: + index = i + + if val > lookup[-1]: + index = len(lookup) - 1 + + return index + + def cart2pol(x, y): """ Convert cartesian coordinates to polar @@ -88,85 +99,258 @@ def pol2cart(rho, phi): return (x, y) -class StarTracker: +def get_star_pdf(r0, f0, geometry, psf, n_pdf_bins, pdf_bin_size, focal_length): + image = np.zeros(len(geometry)) + + psf.update_model_parameters(r0, f0) + + dr = pdf_bin_size * np.rad2deg(np.arctan(1 / focal_length)) / 3600.0 + r = np.linspace( + r0 - dr * n_pdf_bins / 2.0, + r0 + dr * n_pdf_bins / 2.0, + n_pdf_bins, + ) + df = np.deg2rad(pdf_bin_size / 3600.0) * 100 + f = np.linspace( + f0 - df * n_pdf_bins / 2.0, + f0 + df * n_pdf_bins / 2.0, + n_pdf_bins, + ) + + for r_ in r: + for f_ in f: + val = psf.pdf(r_, f_) * dr * df + x, y = pol2cart(r_, f_) + pixelN = geometry.position_to_pix_index(x * u.m, y * u.m) + if pixelN != -1: + image[pixelN] += val + + return image + + +def StarImageGenerator( + self, + radius, + phi, + magnitude, + n_pdf_bins, + pdf_bin_size, + psf_model_name, + psf_model_pars, + camera_name, + focal_length, +): """ - Utility class to provide the position of the star in the telescope's camera frame coordinates at a given time + :param list stars: list of star containers, stars to be placed in image + :param dict psf_model_pars: psf model parameters + """ + camera_geometry = CameraGeometry.from_name(camera_name) + psf = PSFModel.from_name(self.psf_model_type, subarray=self.subarray, parent=self) + psf.update_model_parameters(psf_model_pars) + image = np.zeros(len(camera_geometry)) + for r, p, m in zip(radius, phi, magnitude): + image += m * get_star_pdf( + r, p, camera_geometry, psf, n_pdf_bins, pdf_bin_size, focal_length + ) + + return image + + +class StarTracer: + """ + Utility class to trace a set of stars over a period of time and generate their locations in the camera """ def __init__( self, - star_label, - star_coordinates, - telescope_location, - telescope_focal_length, - telescope_pointing, + stars, + magnitude, + az, + alt, + time, + meteo_parameters, observed_wavelength, - relative_humidity, - temperature, - pressure, - pointing_label=None, + camera_geometry, + focal_length, + location, ): """ - Constructor - - :param str star_label: Star label - :param SkyCoord star_coordinates: Star coordinates in ICRS frame - :param EarthLocation telescope_location: Telescope location coordinates - :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] - :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame - :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] - :param float relative_humidity: Relative humidity - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.hPa] pressure: Pressure [hPa] - :param str pointing_label: Pointing label + param dict stars: dict of Astropy.SkyCoord objects, keys are the nomad labels + param dict magnitude: + param list time: list of Astropy.time objects corresponding to the altitude and azimuth values """ - self.star_label = star_label - self.star_coordinates_icrs = star_coordinates - self.telescope_location = telescope_location - self.telescope_focal_length = telescope_focal_length - self.telescope_pointing = telescope_pointing - self.obswl = observed_wavelength - self.relative_humidity = relative_humidity - self.temperature = temperature - self.pressure = pressure - self.pointing_label = pointing_label - - def position_in_camera_frame(self, timestamp, pointing=None, focal_correction=0): + + self.stars = stars + self.magnitude = magnitude + + self.pointing = PointingInterpolator() + pointing = QTable([az, alt, time], names=["azimuth", "altitude", "time"]) + self.pointing.add_table(0, pointing) + + self.meteo_parameters = meteo_parameters + self.observed_wavelength = observed_wavelength + self.camera_geometry = camera_geometry + self.focal_length = focal_length * u.m + self.location = location + + @classmethod + def from_lookup( + cls, + max_star_magnitude, + az, + alt, + time, + meteo_params, + observed_wavelength, + camera_geometry, + focal_length, + location, + ): + """ + classmethod to use vizier lookup to generate a class instance """ - Calculates star position in the engineering camera frame + _pointing = SkyCoord( + az=az[0], + alt=alt[0], + frame="altaz", + obstime=time[0], + location=location, + obswl=observed_wavelength * u.micron, + relative_humidity=meteo_params["relative_humidity"], + temperature=meteo_params["temperature"] * u.deg_C, + pressure=meteo_params["pressure"] * u.hPa, + ) + stars_in_fov = Vizier.query_region( + _pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] # get all stars that could be in the fov - :param astropy.Time timestamp: Timestamp of the observation - :param SkyCoord pointing: Current telescope pointing in ICRS frame - :param float focal_correction: Correction to the focal length of the telescope. Float, should be provided in meters + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < max_star_magnitude] - :return: Pair (float, float) of star's (x,y) coordinates in the engineering camera frame in meters + stars = {} + magnitude = {} + for star in stars_in_fov: + star_coords = { + star["NOMAD1"]: SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + } + stars.update(star_coords) + magnitude.update({star["NOMAD1"]: star["Bmag"]}) + + return cls( + stars, + magnitude, + az, + alt, + time, + meteo_params, + observed_wavelength, + camera_geometry, + focal_length, + location, + ) + + def get_star_labels(self): """ - # If no telescope pointing is provided, use the telescope pointing, provided - # during the class member initialization - if pointing is None: - pointing = self.telescope_pointing - # Determine current telescope pointing in AltAz - altaz_pointing = pointing.transform_to( - AltAz( - obstime=timestamp, - location=self.telescope_location, - obswl=self.obswl, - relative_humidity=self.relative_humidity, - temperature=self.temperature, - pressure=self.pressure, - ) + Return a list of all stars that are being traced + """ + + return list(self.stars.keys()) + + def get_magnitude(self, star): + """ + Return the magnitude of star + + parameter str star: NOMAD1 label of the star + """ + + return self.magnitude[star] + + def get_pointing(self, t, offset=(0.0, 0.0)): + alt, az = self.pointing(t) + alt += offset[0] * u.rad + az += offset[1] * u.rad + + coords = SkyCoord( + az=az, + alt=alt, + frame="altaz", + obstime=t, + location=self.location, + obswl=self.observed_wavelength * u.micron, + relative_humidity=self.meteo_parameters["relative_humidity"], + temperature=self.meteo_parameters["temperature"] * u.deg_C, + pressure=self.meteo_parameters["pressure"] * u.hPa, ) - # Create current camera frame + + return coords + + def get_camera_frame(self, t, offset=(0.0, 0.0), focal_correction=0.0): + altaz_pointing = self.get_pointing(t, offset=offset) + camera_frame = EngineeringCameraFrame( telescope_pointing=altaz_pointing, - focal_length=self.telescope_focal_length + focal_correction * u.m, - obstime=timestamp, - location=self.telescope_location, + focal_length=self.focal_length + focal_correction * u.m, + obstime=t, + location=self.location, ) - # Calculate the star's coordinates in the current camera frame - star_coords_camera = self.star_coordinates_icrs.transform_to(camera_frame) - return (star_coords_camera.x.to_value(), star_coords_camera.y.to_value()) + + return camera_frame + + def get_current_geometry(self, t, offset=(0.0, 0.0), focal_correction=0.0): + camera_frame = self.get_camera_frame( + t, offset=offset, focal_correction=focal_correction + ) + + current_geometry = self.camera_geometry.transform_to(camera_frame) + + return current_geometry + + def get_position_in_camera(self, t, star, offset=(0.0, 0.0), focal_correction=0.0): + camera_frame = self.get_camera_frame( + t, offset=offset, focal_correction=focal_correction + ) + # Calculate the stars coordinates in the current camera frame + coords = self.stars[star].transform_to(camera_frame) + return (coords.x.to_value(), coords.y.to_value()) + + def get_position_in_pixel(self, t, star, focal_correction=0.0): + x, y = self.get_position_in_camera(t, star, focal_correction=focal_correction) + current_geometry = self.get_current_geometry(t) + + return current_geometry.position_to_pix_index(x, y) + + def get_expected_star_pixels(self, t, focal_correction=0.0): + """ + Determine which in which pixels stars are expected for a series of images + + Parameters + ---------- + t : list + list of time values where the images were captured + """ + + res = [] + + for star in self.get_star_labels(): + expected_central_pixel = self.get_positions_in_pixel( + t, star, focal_correction=focal_correction + ) + cluster = copy.deepcopy( + self.camera_geometry.neighbors[expected_central_pixel] + ) # get the neighborhood of the star + cluster_corona = [] + + for pixel_index in cluster: + cluster_corona.extend( + copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) + ) # and add another layer of pixels to be sure + + cluster.extend(cluster_corona) + cluster.append(expected_central_pixel) + res.extend(list(set(cluster))) + + return res class PointingCalculator(TelescopeComponent): @@ -181,6 +365,12 @@ class PointingCalculator(TelescopeComponent): The location of the telescope for which the pointing correction is to be calculated """ + stats_extractor = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="Plain"), + default_value="Plain", + help="Name of the StatisticsExtractor Subclass to be used.", + ).tag(config=True) + telescope_location = Dict( {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, help="Telescope location, longitude and latitude should be expressed in deg, " @@ -200,7 +390,7 @@ class PointingCalculator(TelescopeComponent): ).tag(config=True) max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + 7.0, help="Maximal magnitude of the star to be considered in the analysis" ).tag(config=True) cleaning = Dict( @@ -213,7 +403,7 @@ class PointingCalculator(TelescopeComponent): ).tag(config=True) psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + trait=ComponentName(PSFModel, default_value="ComaModel"), default_value="ComaModel", help="Name of the PSFModel Subclass to be used.", ).tag(config=True) @@ -223,16 +413,22 @@ class PointingCalculator(TelescopeComponent): help="Meteorological parameters in [dimensionless, deg C, hPa]", ).tag(config=True) + n_pdf_bins = Integer(1000, help="Camera focal length").tag(config=True) + + pdf_bin_size = Float(10.0, help="Camera focal length").tag(config=True) + + focal_length = Float(1.0, help="Camera focal length in meters").tag(config=True) + def __init__( self, subarray, + geometry, config=None, parent=None, **kwargs, ): super().__init__( subarray=subarray, - stats_extractor="Plain", config=config, parent=parent, **kwargs, @@ -247,240 +443,110 @@ def __init__( lat=self.telescope_location["latitude"] * u.deg, height=self.telescope_location["elevation"] * u.m, ) - - def __call__(self, input_url, tel_id): - self.tel_id = tel_id - - if self._check_req_data(input_url, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - # first get the camera geometry and pointing for the file and determine what stars we should see - - with EventSource(input_url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) # get some pointing to make a list of stars that we expect to see - - self.pointing = self.pointing.transform_to("icrs") - - self.broken_pixels = np.unique(np.where(self.broken_pixels)) - - self.image_size = len( - event.variance_image.image - ) # get the size of images of the camera we are calibrating - - self.stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] # get all stars that could be in the fov - - self.stars_in_fov = self.stars_in_fov[ - self.tars_in_fov["Bmag"] < self.max_star_magnitude - ] # select stars for magnitude to exclude those we would not be able to see - - # get the accumulated variance images - - ( - accumulated_pointing, - accumulated_times, - variance_statistics, - ) = self._get_accumulated_images(input_url) - - accumulated_images = np.array([x.mean for x in variance_statistics]) - - star_pixels = self._get_expected_star_pixels( - accumulated_times, accumulated_pointing + self.stats_aggregator = StatisticsExtractor.from_name( + self.stats_extractor, subarray=self.subarray, parent=self ) - star_mask = np.ones(self.image_size, dtype=bool) - - star_mask[star_pixels] = False - - # get NSB values - - nsb = np.mean(accumulated_images[star_mask], axis=1) - nsb_std = np.std(accumulated_images[star_mask], axis=1) - - clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) - - reco_stars = [] - - for i, image in enumerate(clean_images): - reco_stars.append([]) - camera_frame = EngineeringCameraFrame( - telescope_pointing=accumulated_pointing[i], - focal_length=self.focal_length, - obstime=accumulated_times[i].utc, - location=self.location, - ) - for star in self.stars_in_fov: - reco_stars[-1].append( - self._fit_star_position( - star, accumulated_times[i], camera_frame, image, nsb_std[i] - ) - ) + self.set_camera(geometry) - # now fit the pointing corrections. - correction = [] - for i, stars in enumerate(reco_stars): - fitter = Pointing_Fitter( - stars, - accumulated_times[i], - accumulated_pointing[i], - self.location, - self.focal_length, - self.observed_wavelength, - self.meteo_parameters, - ) + def set_camera(self, geometry, focal_lengh): + if isinstance(geometry, str): + self.camera_geometry = CameraGeometry.from_name(geometry) - correction.append(fitter.fit()) + self.pixel_radius = self.camera_geometry.pixel_width[0] - return correction - - def _check_req_data(self, url, calibration_type): + def ingest_data(self, data_table): """ - Check if the prerequisite calibration data exists in the files - - Parameters + Attributes ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain + data_table : Table + Table containing a series of variance images with corresponding initial pointing values, trigger times and calibration data """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - calibration_data = getattr(event.mon.tel[self.tel_id], calibration_type) - - if calibration_data is None: - return False - - return True + # set up the StarTracer here to track stars in the camera + self.tracer = StarTracer.from_lookup( + data_table["telescope_pointing_azimuth"], + data_table["telescope_pointing_altitude"], + data_table["time"], + self.meteo_parameters, + self.observed_wavelength, + self.camera_geometry, + self.focal_length, + self.telescope_location, + ) - def _calibrate_var_images(self, var_images, time, calibration_file): - """ - Calibrate a set of variance images + self.broken_pixels = np.unique(np.where(data_table["unusable_pixels"])) - Parameters - ---------- - var_images : list - list of variance images - time : list - list of times correxponding to the variance images - calibration_file : str - name of the file where the calibration data can be found - """ - # So i need to use the interpolator classes to read the calibration data - relative_gains = FlatFieldInterpolator( - calibration_file - ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square(relative_gains(time[i])), + for azimuth, altitude, time in zip( + data_table["telescope_pointing_azimuth"], + data_table["telescope_pointing_altitude"], + data_table["time"], + ): + _pointing = SkyCoord( + az=azimuth, + alt=altitude, + frame="altaz", + obstime=time, + location=self.location, + obswl=self.observed_wavelength * u.micron, + relative_humidity=self.meteo_parameters["relative_humidity"], + temperature=self.meteo_parameters["temperature"] * u.deg_C, + pressure=self.meteo_parameters["pressure"] * u.hPa, ) + self.pointing.append(_pointing) - return var_images + self.image_size = len( + data_table["variance_images"][0].image + ) # get the size of images of the camera we are calibrating - def _get_expected_star_pixels(self, time_list, pointing_list): - """ - Determine which in which pixels stars are expected for a series of images + # get the accumulated variance images - Parameters - ---------- - time_list : list - list of time values where the images were captured - pointing_list : list - list of pointing values for the images - """ + self._get_accumulated_images(data_table) - res = [] + def fit_stars(self): + stars = self.tracer.get_star_labels() - for pointing, time in zip( - pointing_list, time_list - ): # loop over time and pointing of images - temp = [] + self.all_containers = [] - camera_frame = EngineeringCameraFrame( - telescope_pointing=pointing, - focal_length=self.focal_length, - obstime=time.utc, - location=self.location, - ) # get the engineering camera frame for the pointing + for t, image in (self.accumulated_times, self.accumulated_images): + self.all_containers.append([]) - for star in self.stars_in_fov: - star_coords = SkyCoord( - star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" - ) - star_coords = star_coords.transform_to(camera_frame) - expected_central_pixel = self.camera_geometry.transform_to( - camera_frame - ).position_to_pix_index( - star_coords.x, star_coords.y - ) # get where the star should be - cluster = copy.deepcopy( - self.camera_geometry.neighbors[expected_central_pixel] - ) # get the neighborhood of the star - cluster_corona = [] - - for pixel_index in cluster: - cluster_corona.extend( - copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) - ) # and add another layer of pixels to be sure - - cluster.extend(cluster_corona) - cluster.append(expected_central_pixel) - temp.extend(list(set(cluster))) - - res.append(temp) + for star in stars: + container = self._fit_star_position(star, t, image, self.nsb_std) - return res + self.all_containers[-1].append(container) - def _fit_star_position( - self, star, timestamp, camera_frame, image, nsb_std, current_pointing - ): - star_coords = SkyCoord( - star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" - ) + return self.all_containers - star_coords = star_coords.transform_to(camera_frame) + def _fit_star_position(self, star, t, image, nsb_std): + x, y = self.tracer.get_position_in_camera(self, t, star) - rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) + rho, phi = cart2pol(x, y) if phi < 0: phi = phi + 2 * np.pi star_container = StarContainer( - label=star["NOMAD1"], - magnitude=star["Bmag"], - expected_x=star_coords.x, - expected_y=star_coords.y, + label=star, + magnitude=self.tracer.get_magnitude(star), + expected_x=x, + expected_y=y, expected_r=rho * u.m, expected_phi=phi * u.rad, - timestamp=timestamp, + timestamp=t, ) - current_geometry = self.camera_geometry.transform_to(camera_frame) + current_geometry = self.tracer.get_current_geometry(t) - hit_pdf = self._get_star_pdf(star, current_geometry) + hit_pdf = get_star_pdf( + rho, + phi, + current_geometry, + self.psf, + self.n_pdf_bins, + self.pdf_bin_size, + self.focal_length.to_value(u.m), + ) cluster = np.where(hit_pdf > self.pdf_percentile_limit * np.sum(hit_pdf)) if not np.any(image[cluster] > self.min_star_prominence * nsb_std): @@ -552,47 +618,11 @@ def _fit_star_position( return star_container - def _get_accumulated_images(self, input_url): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=self.tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=True, - ) - - # get the trigger type for all images and make a mask - - event_mask = dl1_table["event_type"] == 2 - - # get the pointing for all images and filter for trigger type - - altitude = dl1_table["telescope_pointing_altitude"][event_mask] - azimuth = dl1_table["telescope_pointing_azimuth"][event_mask] - time = dl1_table["time"][event_mask] - - pointing = [ - SkyCoord( - az=x, alt=y, frame="altaz", obstime=z.tai.utc, location=self.location - ) - for x, y, z in zip(azimuth, altitude, time) - ] - - # get the time and images from the data - - variance_images = copy.deepcopy(dl1_table["variance_image"][event_mask]) + def _get_accumulated_images(self, data_table): + variance_images = data_table["variance_images"] # now make a filter to to reject EAS light and starlight and keep a separate EAS filter - charge_images = dl1_table["image"][event_mask] - light_mask = [ tailcuts_clean( self.camera_geometry, @@ -600,12 +630,14 @@ def _get_accumulated_images(self, input_url): picture_thresh=self.cleaning["pic_thresh"], boundary_thresh=self.cleaning["bound_thresh"], ) - for x in charge_images + for x in data_table["charge_image"] ] shower_mask = copy.deepcopy(light_mask) - star_pixels = self._get_expected_star_pixels(time, pointing) + star_pixels = [ + self.tracer.get_expected_star_pixels(t) for t in data_table["time"] + ] light_mask[:, star_pixels] = True @@ -620,163 +652,46 @@ def _get_accumulated_images(self, input_url): # now calibrate the images - variance_images = self._calibrate_var_images( - self, variance_images, time, input_url - ) + relative_gains = FlatFieldInterpolator() + relative_gains.add_table(0, data_table["relative_gain"]) - # Get the average variance across the data to + for i, var_image in enumerate(variance_images): + variance_images[i] = np.divide( + var_image, + np.square(relative_gains(data_table["time"][i]).median), + ) # then turn it into a table that the extractor can read - variance_image_table = QTable([time, variance_images], names=["time", "image"]) - - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[self.tel_id]] + variance_image_table = QTable( + [data_table["time"], variance_images], names=["time", "image"] + ) # get the cumulative variance images using the statistics extractor and return the value - variance_statistics = extractor(variance_image_table) - - accumulated_times = np.array([x.validity_start for x in variance_statistics]) - - # calculate where stars might be - - accumulated_pointing = np.array( - [x for x in pointing if pointing.time in accumulated_times] + variance_statistics = self.stats_aggregator( + variance_image_table, col_name="image" ) - return (accumulated_pointing, accumulated_times, variance_statistics) - - def _get_star_pdf(self, star, current_geometry): - image = np.zeros(self.image_size) - - r0 = star.expected_r.to_value(u.m) - f0 = star.expected_phi.to_value(u.rad) - - self.psf_model.update_model_parameters(r0, f0) - - dr = ( - self.pdf_bin_size - * np.rad2deg(np.arctan(1 / self.focal_length.to_value(u.m))) - / 3600.0 - ) - r = np.linspace( - r0 - dr * self.n_pdf_bins / 2.0, - r0 + dr * self.n_pdf_bins / 2.0, - self.n_pdf_bins, + self.accumulated_times = np.array( + [x.validity_start for x in variance_statistics] ) - df = np.deg2rad(self.pdf_bin_size / 3600.0) * 100 - f = np.linspace( - f0 - df * self.n_pdf_bins / 2.0, - f0 + df * self.n_pdf_bins / 2.0, - self.n_pdf_bins, - ) - - for r_ in r: - for f_ in f: - val = self.psf_model.pdf(r_, f_) * dr * df - x, y = pol2cart(r_, f_) - pixelN = current_geometry.position_to_pix_index(x * u.m, y * u.m) - if pixelN != -1: - image[pixelN] += val - - return image + accumulated_images = np.array([x.mean for x in variance_statistics]) -class Pointing_Fitter: - """ - Pointing correction fitter - """ + star_pixels = [ + self.tracer.get_expected_star_pixels(t) for t in data_table["time"] + ] - def __init__( - self, - stars, - time, - telescope_pointing, - telescope_location, - focal_length, - observed_wavelength, - meteo_params, - fit_grid="polar", - ): - """ - Constructor - - :param list stars: List of lists of star containers the first dimension is supposed to be time - :param list time: List of time values for the when the star locations were fitted - :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame - :param EarthLocation telescope_location: Telescope location - :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] - :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] - :param float relative_humidity: Relative humidity - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.hPa] pressure: Pressure [hPa] - :param str fit_grid: Coordinate system grid to use. Either polar or cartesian - """ - self.star_containers = stars - self.time = time - self.telescope_pointing = telescope_pointing - self.telescope_location = telescope_location - self.focal_length = focal_length - self.obswl = observed_wavelength - self.relative_humidity = meteo_params["relative_humidity"] - self.temperature = meteo_params["temperature"] - self.pressure = meteo_params["pressure"] - self.stars = [] - self.data = [] - self.errors = [] - # Construct the data here. Add only stars that were found - for star in stars: - if not np.isnan(star.reco_x): - self.data.append(star.reco_x) - self.data.append(star.reco_y) - self.errors.append(star.reco_dx) - self.errors.append(star.reco_dy) - self.stars.append(self.init_star(star.label)) - self.fit_mode = "xy" - self.fit_grid = fit_grid - self.star_motion_model = Model(self.fit_function) - self.fit_summary = None - self.fit_resuts = None - - def init_star(self, star_label): - """ - Initialize StarTracker object for a given star + star_mask = np.ones(self.image_size, dtype=bool) - :param str star_label: Star label according to NOMAD catalog + star_mask[star_pixels] = False - :return: StarTracker object - """ - star = Vizier(catalog="NOMAD").query_constraints(NOMAD1=star_label)[0] - star_coords = SkyCoord( - star["RAJ2000"], - star["DEJ2000"], - unit="deg", - frame="icrs", - obswl=self.obswl, - relative_humidity=self.relative_humidity, - temperature=self.temperature, - pressure=self.pressure, - ) - st = StarTracker( - star_label, - star_coords, - self.telescope_location, - self.focal_length, - self.telescope_pointing, - self.obswl, - self.relative_humidity, - self.temperature, - self.pressure, - ) - return st + # get NSB values - def current_pointing(self, t): - """ - Retrieve current telescope pointing - """ - index = self.times.index(t) + nsb = np.mean(accumulated_images[star_mask], axis=1) + self.nsb_std = np.std(accumulated_images[star_mask], axis=1) - return self.telescope_pointing[index] + self.clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) def fit_function(self, p, t): """ @@ -787,81 +702,48 @@ def fit_function(self, p, t): """ - time = Time(t, format="unix_tai", scale="utc") coord_list = [] - m_ra, m_dec = p - new_ra = self.current_pointing(t).ra + m_ra * u.deg - new_dec = self.current_pointing(t).dec + m_dec * u.deg - - new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) - - m_ra, m_dec = p - new_ra = self.current_pointing(t).ra + m_ra * u.deg - new_dec = self.current_pointing(t).dec + m_dec * u.deg - new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) - for star in self.stars: - x, y = star.position_in_camera_frame(time, new_pointing) - if self.fit_grid == "polar": - x, y = cart2pol(x, y) - coord_list.extend([x]) - coord_list.extend([y]) + index = get_index_step( + t, self.accumulated_times + ) # this gives you the index corresponding to the + for star in self.all_containers[index]: + if not np.isnan(star.reco_x): + x, y = self.tracer.get_position_in_camera(star.label, t, offset=p) + coord_list.extend([x]) + coord_list.extend([y]) return coord_list - def fit(self, fit_mode="xy"): + def fit(self): """ Performs the ODR fit of stars trajectories and saves the results as self.fit_results :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. """ - self.fit_mode = fit_mode - if self.fit_mode == "radec" or self.fit_mode == "xy": + + results = [] + for i, t in enumerate(self.accumulated_times): init_mispointing = [0, 0] - elif self.fit_mode == "y": - init_mispointing = [0] - elif self.fit_mode == "xyz": - init_mispointing = [0, 0, 0] - if self.errors is not None: - rdata = RealData(x=self.time, y=self.data, sy=self.errors) - else: - rdata = RealData(x=self.time, y=self.data) - odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) - self.fit_summary = odr.run() - if self.fit_mode == "radec": - self.fit_results = pd.DataFrame( - data={ - "dRA": [self.fit_summary.beta[0]], - "dDEC": [self.fit_summary.beta[1]], - "eRA": [self.fit_summary.sd_beta[0]], - "eDEC": [self.fit_summary.sd_beta[1]], - } - ) - elif self.fit_mode == "xy": - self.fit_results = pd.DataFrame( - data={ - "dX": [self.fit_summary.beta[0]], - "dY": [self.fit_summary.beta[1]], - "eX": [self.fit_summary.sd_beta[0]], - "eY": [self.fit_summary.sd_beta[1]], - } - ) - elif self.fit_mode == "y": - self.fit_results = pd.DataFrame( - data={ - "dY": [self.fit_summary.beta[0]], - "eY": [self.fit_summary.sd_beta[0]], - } - ) - elif self.fit_mode == "xyz": - self.fit_results = pd.DataFrame( + data = [] + errors = [] + for star in self.all_containers: + if not np.isnan(star.reco_x): + data.append(star.reco_x) + data.append(star.reco_y) + errors.append(star.reco_dx) + errors.append(star.reco_dy) + + rdata = RealData(x=[t], y=data, sy=self.errors) + odr = ODR(rdata, self.fit_function, beta0=init_mispointing) + fit_summary = odr.run() + fit_results = pd.DataFrame( data={ - "dX": [self.fit_summary.beta[0]], - "dY": [self.fit_summary.beta[1]], - "dZ": [self.fit_summary.beta[2]], - "eX": [self.fit_summary.sd_beta[0]], - "eY": [self.fit_summary.sd_beta[1]], - "eZ": [self.fit_summary.sd_beta[2]], + "dAZ": [fit_summary.beta[0]], + "dALT": [fit_summary.beta[1]], + "eAZ": [fit_summary.sd_beta[0]], + "eALT": [fit_summary.sd_beta[1]], } ) - return self.fit_results + results.append(fit_results) + return results diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 458070b8145..4bea11fd464 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -8,7 +8,6 @@ import numpy as np from scipy.stats import laplace, laplace_asymmetric -from traitlets import List class PSFModel: @@ -39,6 +38,10 @@ def from_name(cls, name, **kwargs): def pdf(self, *args): pass + @abstractmethod + def update_location(self, *args): + pass + @abstractmethod def update_model_parameters(self, *args): pass @@ -49,18 +52,23 @@ class ComaModel(PSFModel): PSF model, describing pure coma aberrations PSF effect """ - asymmetry_params = List( - default_value=[0.49244797, 9.23573115, 0.15216096], - help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", - ).tag(config=True) - radial_scale_params = List( - default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], - help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", - ).tag(config=True) - az_scale_params = List( - default_value=[0.24271557, 7.5511501, 0.02037972], - help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", - ).tag(config=True) + def __init__( + self, + asymmetry_params=[0.49244797, 9.23573115, 0.15216096], + radial_scale_params=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + az_scale_params=[0.24271557, 7.5511501, 0.02037972], + ): + """ + PSF model, describing pure coma aberrations PSF effect + + param list asymmetry_params Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera + param list radial_scale_params Parameters describing the dependency of the radial scale on the distance to the center of the camera + param list radial_scale_params Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera + """ + + self.asymmetry_params = asymmetry_params + self.radial_scale_params = radial_scale_params + self.az_scale_params = az_scale_params def k_func(self, x): return ( @@ -87,7 +95,21 @@ def pdf(self, r, f): f, *self.azimuthal_pdf_params ) - def update_model_parameters(self, r, f): + def update_model_parameters(self, model_params): + if not ( + model_params["asymmetry_params"] == 3 + and model_params["radial_scale_params"] == 4 + and model_params["az_scale_params"] == 3 + ): + raise ValueError( + "asymmetry_params and az_scale_params needs to have length 3 and radial_scale_params length 4" + ) + + self.asymmetry_params = model_params["asymmetry_params"] + self.radial_scale_params = model_params["radial_scale_params"] + self.az_scale_params = model_params["az_scale_params"] + + def update_location(self, r, f): k = self.k_func(r) sr = self.sr_func(r) sf = self.sf_func(r) From 4b975444c01c630b6207ca0df5efea39332bc214 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 15:00:33 +0200 Subject: [PATCH 105/221] Adding a placeholder test script --- src/ctapipe/calib/camera/tests/test_pointing.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/ctapipe/calib/camera/tests/test_pointing.py diff --git a/src/ctapipe/calib/camera/tests/test_pointing.py b/src/ctapipe/calib/camera/tests/test_pointing.py new file mode 100644 index 00000000000..c29014190cb --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_pointing.py @@ -0,0 +1,3 @@ +""" +Tests for StatisticsExtractor and related functions +""" From 40c39689338bfd5ea36f3b42e6d5516a4a1cc96e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 17:23:17 +0200 Subject: [PATCH 106/221] fixed astroquery import --- src/ctapipe/calib/camera/pointing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index a50f1e4dcf1..d36e1a5b9cd 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -9,9 +9,9 @@ import astropy.units as u import numpy as np import pandas as pd -import Vizier # discuss this dependency with max etc. from astropy.coordinates import Angle, EarthLocation, SkyCoord from astropy.table import QTable +from astroquery.vizier import Vizier # discuss this dependency with max etc. from scipy.odr import ODR, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor From cea1f219d0c842c719cc10bf85f52d149ba99327 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 1 Oct 2024 09:15:30 +0200 Subject: [PATCH 107/221] removing merge marks --- src/ctapipe/calib/camera/calibrator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6ad47b525fe..baf3d2f1057 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,10 +2,6 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ -<<<<<<< HEAD - -======= ->>>>>>> c5f385f0 (I added a basic star fitter) from functools import cache import astropy.units as u @@ -23,9 +19,9 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.reducer import DataVolumeReducer - __all__ = ["CameraCalibrator"] + @cache def _get_pixel_index(n_pixels): """Cached version of ``np.arange(n_pixels)``""" @@ -50,6 +46,7 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill From c6d7245c81c50fc95dec9039ed8cebff4f27921e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 1 Oct 2024 14:21:38 +0200 Subject: [PATCH 108/221] I moved the interpolator classes for flatfield events to to their proper place --- src/ctapipe/calib/camera/extractor.py | 233 --------------- src/ctapipe/calib/camera/pointing.py | 26 +- src/ctapipe/io/interpolation.py | 369 ------------------------ src/ctapipe/monitoring/interpolation.py | 187 ++++++++++++ 4 files changed, 200 insertions(+), 615 deletions(-) delete mode 100644 src/ctapipe/calib/camera/extractor.py delete mode 100644 src/ctapipe/io/interpolation.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py deleted file mode 100644 index 7093d057f20..00000000000 --- a/src/ctapipe/calib/camera/extractor.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -Extraction algorithms to compute the statistics from a sequence of images -""" - -__all__ = [ - "StatisticsExtractor", - "PlainExtractor", - "SigmaClippingExtractor", -] - -from abc import abstractmethod - -import numpy as np -from astropy.stats import sigma_clipped_stats - -from ctapipe.containers import StatisticsContainer -from ctapipe.core import TelescopeComponent -from ctapipe.core.traits import ( - Int, - List, -) - - -class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) - image_median_cut_outliers = List( - [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", - ).tag(config=True) - image_std_cut_outliers = List( - [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", - ).tag(config=True) - - def __init__(self, subarray, config=None, parent=None, **kwargs): - """ - Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). - - Parameters - ---------- - kwargs - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - @abstractmethod - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - """ - Call the relevant functions to extract the statistics - for the particular extractor. - - Parameters - ---------- - dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape - (n_images, n_channels, n_pix). - col_name : string - column name in the dl1 table - - Returns - ------- - List StatisticsContainer: - List of extracted statistics and validity ranges - """ - - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _plain_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median over the sample per pixel - pixel_median = np.ma.median(masked_images, axis=0) - - # mean over the sample per pixel - pixel_mean = np.ma.mean(masked_images, axis=0) - - # std over the sample per pixel - pixel_std = np.ma.std(masked_images, axis=0) - - # median of the median over the camera - # median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # outliers from median - image_median_outliers = np.logical_or( - pixel_median < self.image_median_cut_outliers[0], - pixel_median > self.image_median_cut_outliers[1], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - ) - - -class SigmaClippingExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using astropy's sigma clipping functions - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median of the event images - # image_median = np.ma.median(masked_images, axis=-1) - - # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - # mask pixels without defined statistical values - pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) - pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) - pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - - # only warn for values discard in the sigma clipping, not those from before - # outliers = unused_values & (~masked_images.mask) - - # add outliers identified by sigma clipping for following operations - masked_images.mask |= unused_values - - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # median of the std over the camera - median_of_pixel_std = np.ma.median(pixel_std, axis=1) - - # std of the std over camera - std_of_pixel_std = np.ma.std(pixel_std, axis=1) - - # outliers from median - image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = np.logical_or( - image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], - image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], - ) - - # outliers from standard deviation - deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = np.logical_or( - deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - std_outliers=image_std_outliers.filled(True), - ) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index d36e1a5b9cd..d5673b17693 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -14,7 +14,6 @@ from astroquery.vizier import Vizier # discuss this dependency with max etc. from scipy.odr import ODR, RealData -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.core import TelescopeComponent @@ -28,7 +27,8 @@ from ctapipe.image import tailcuts_clean from ctapipe.image.psf_model import PSFModel from ctapipe.instrument import CameraGeometry -from ctapipe.io import FlatFieldInterpolator, PointingInterpolator +from ctapipe.monitoring.aggregator import StatisticsAggregator +from ctapipe.monitoring.interpolation import FlatFieldInterpolator, PointingInterpolator __all__ = ["PointingCalculator", "StarImageGenerator"] @@ -359,16 +359,16 @@ class PointingCalculator(TelescopeComponent): Attributes ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + stats_aggregator: str + The name of the StatisticsAggregator subclass to be used to calculate the statistics of an image set telescope_location: dict The location of the telescope for which the pointing correction is to be calculated """ - stats_extractor = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="Plain"), - default_value="Plain", - help="Name of the StatisticsExtractor Subclass to be used.", + stats_aggregator = TelescopeParameter( + trait=ComponentName(StatisticsAggregator, default_value="PlainAggregator"), + default_value="PlainAggregator", + help="Name of the StatisticsAggregator Subclass to be used.", ).tag(config=True) telescope_location = Dict( @@ -444,8 +444,8 @@ def __init__( height=self.telescope_location["elevation"] * u.m, ) - self.stats_aggregator = StatisticsExtractor.from_name( - self.stats_extractor, subarray=self.subarray, parent=self + self.image_aggregator = StatisticsAggregator.from_name( + self.stats_aggregator, subarray=self.subarray, parent=self ) self.set_camera(geometry) @@ -662,14 +662,14 @@ def _get_accumulated_images(self, data_table): np.square(relative_gains(data_table["time"][i]).median), ) - # then turn it into a table that the extractor can read + # then turn it into a table that the aggregator can read variance_image_table = QTable( [data_table["time"], variance_images], names=["time", "image"] ) - # get the cumulative variance images using the statistics extractor and return the value + # get the cumulative variance images using the statistics aggregator and return the value - variance_statistics = self.stats_aggregator( + variance_statistics = self.image_aggregator( variance_image_table, col_name="image" ) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py deleted file mode 100644 index e0e27470c99..00000000000 --- a/src/ctapipe/io/interpolation.py +++ /dev/null @@ -1,369 +0,0 @@ -from abc import ABCMeta, abstractmethod -from typing import Any - -import astropy.units as u -import numpy as np -import tables -from astropy.time import Time -from scipy.interpolate import interp1d - -from ctapipe.core import Component, traits - -from .astropy_helpers import read_table - - -class ChunkFunction: - - """ - Chunk Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the latest starting - valid chunk is given or the earliest available still valid chunk for any - pixels without valid data. - - Parameters - ---------- - values : None | np.array - Numpy array of the data that is to be interpolated. - The first dimension needs to be an index over time - times : None | np.array - Time values over which data are to be interpolated - need to be sorted and have same length as first dimension of values - """ - - def __init__( - self, - start_times, - end_times, - values, - bounds_error=True, - fill_value="extrapolate", - assume_sorted=True, - copy=False, - ): - self.values = values - self.start_times = start_times - self.end_times = end_times - self.bounds_error = bounds_error - self.fill_value = fill_value - - def __call__(self, point): - if point < self.start_times[0]: - if self.bounds_error: - raise ValueError("below the interpolation range") - - if self.fill_value == "extrapolate": - return self.values[0] - - else: - a = np.empty(self.values[0].shape) - a[:] = np.nan - return a - - elif point > self.end_times[-1]: - if self.bounds_error: - raise ValueError("above the interpolation range") - - if self.fill_value == "extrapolate": - return self.values[-1] - - else: - a = np.empty(self.values[0].shape) - a[:] = np.nan - return a - - else: - i = np.searchsorted( - self.start_times, point, side="left" - ) # Latest valid chunk - j = np.searchsorted( - self.end_times, point, side="left" - ) # Earliest valid chunk - return np.where( - np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] - ) # Give value for latest chunk unless its nan. If nan give earliest chunk value - - -class Interpolator(Component, metaclass=ABCMeta): - """ - Interpolator parent class. - - Parameters - ---------- - h5file : None | tables.File - A open hdf5 file with read access. - """ - - bounds_error = traits.Bool( - default_value=True, - help="If true, raises an exception when trying to extrapolate out of the given table", - ).tag(config=True) - - extrapolate = traits.Bool( - help="If bounds_error is False, this flag will specify whether values outside" - "the available values are filled with nan (False) or extrapolated (True).", - default_value=False, - ).tag(config=True) - - telescope_data_group = None - required_columns = set() - expected_units = {} - - def __init__(self, h5file=None, **kwargs): - super().__init__(**kwargs) - - if h5file is not None and not isinstance(h5file, tables.File): - raise TypeError("h5file must be a tables.File") - self.h5file = h5file - - self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) - if self.bounds_error: - self.interp_options["bounds_error"] = True - elif self.extrapolate: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = "extrapolate" - else: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = np.nan - - self._interpolators = {} - - @abstractmethod - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - This method reads input tables and creates instances of the needed interpolators - to be added to _interpolators. The first index of _interpolators needs to be - tel_id, the second needs to be the name of the parameter that is to be interpolated - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are always ``time`` as ``Time`` column and - other columns for the data that is to be interpolated - """ - - pass - - def _check_tables(self, input_table): - missing = self.required_columns - set(input_table.colnames) - if len(missing) > 0: - raise ValueError(f"Table is missing required column(s): {missing}") - for col in self.expected_units: - unit = input_table[col].unit - if unit is None: - if self.expected_units[col] is not None: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - elif not self.expected_units[col].is_equivalent(unit): - if self.expected_units[col] is None: - raise ValueError(f"{col} must have units compatible with 'None'") - else: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - - def _check_interpolators(self, tel_id): - if tel_id not in self._interpolators: - if self.h5file is not None: - self._read_parameter_table(tel_id) # might need to be removed - else: - raise KeyError(f"No table available for tel_id {tel_id}") - - def _read_parameter_table(self, tel_id): - input_table = read_table( - self.h5file, - f"{self.telescope_data_group}/tel_{tel_id:03d}", - ) - self.add_table(tel_id, input_table) - - -class PointingInterpolator(Interpolator): - """ - Interpolator for pointing and pointing correction data - """ - - telescope_data_group = "/dl0/monitoring/telescope/pointing" - required_columns = frozenset(["time", "azimuth", "altitude"]) - expected_units = {"azimuth": u.rad, "altitude": u.rad} - - def __call__(self, tel_id, time): - """ - Interpolate alt/az for given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the pointing - - Returns - ------- - altitude : astropy.units.Quantity[deg] - interpolated altitude angle - azimuth : astropy.units.Quantity[deg] - interpolated azimuth angle - """ - - self._check_interpolators(tel_id) - - mjd = time.tai.mjd - az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) - alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) - return alt, az - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` - as quantity columns for pointing and pointing correction data. - """ - - self._check_tables(input_table) - - if not isinstance(input_table["time"], Time): - raise TypeError("'time' column of pointing table must be astropy.time.Time") - - input_table = input_table.copy() - input_table.sort("time") - - az = input_table["azimuth"].quantity.to_value(u.rad) - # prepare azimuth for interpolation by "unwrapping": i.e. turning - # [359, 1] into [359, 361]. This assumes that if we get values like - # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees - # the other way around. This should be true for all telescopes given - # the sampling speed of pointing values and their maximum movement speed. - # No telescope can turn more than 180° in 2 seconds. - az = np.unwrap(az) - alt = input_table["altitude"].quantity.to_value(u.rad) - mjd = input_table["time"].tai.mjd - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) - self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) - - -class FlatFieldInterpolator(Interpolator): - """ - Interpolator for flatfield data - """ - - telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["start_time", "end_time", "gain"]) - expected_units = {"gain": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate flatfield data for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - ffield : array [float] - interpolated flatfield data - """ - - self._check_interpolators(tel_id) - - ffield = self._interpolators[tel_id]["gain"](time) - return ffield - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "gain" - for the flatfield data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("start_time") - start_time = input_table["start_time"] - end_time = input_table["end_time"] - gain = input_table["gain"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = ChunkFunction( - start_time, end_time, gain, **self.interp_options - ) - - -class PedestalInterpolator(Interpolator): - """ - Interpolator for Pedestal data - """ - - telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["start_time", "end_time", "pedestal"]) - expected_units = {"pedestal": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate pedestal or gain for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - pedestal : array [float] - interpolated pedestal values - """ - - self._check_interpolators(tel_id) - - pedestal = self._interpolators[tel_id]["pedestal"](time) - return pedestal - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "pedestal" - for the pedestal data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("start_time") - start_time = input_table["start_time"] - end_time = input_table["end_time"] - pedestal = input_table["pedestal"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = ChunkFunction( - start_time, end_time, pedestal, **self.interp_options - ) diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py index 84064cbc1a3..caceccd8c43 100644 --- a/src/ctapipe/monitoring/interpolation.py +++ b/src/ctapipe/monitoring/interpolation.py @@ -15,6 +15,77 @@ ] +class ChunkFunction: + + """ + Chunk Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the latest starting + valid chunk is given or the earliest available still valid chunk for any + pixels without valid data. + + Parameters + ---------- + values : None | np.array + Numpy array of the data that is to be interpolated. + The first dimension needs to be an index over time + times : None | np.array + Time values over which data are to be interpolated + need to be sorted and have same length as first dimension of values + """ + + def __init__( + self, + start_times, + end_times, + values, + bounds_error=True, + fill_value="extrapolate", + assume_sorted=True, + copy=False, + ): + self.values = values + self.start_times = start_times + self.end_times = end_times + self.bounds_error = bounds_error + self.fill_value = fill_value + + def __call__(self, point): + if point < self.start_times[0]: + if self.bounds_error: + raise ValueError("below the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[0] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + elif point > self.end_times[-1]: + if self.bounds_error: + raise ValueError("above the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[-1] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + else: + i = np.searchsorted( + self.start_times, point, side="left" + ) # Latest valid chunk + j = np.searchsorted( + self.end_times, point, side="left" + ) # Earliest valid chunk + return np.where( + np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] + ) # Give value for latest chunk unless its nan. If nan give earliest chunk value + + class Interpolator(Component, metaclass=ABCMeta): """ Interpolator parent class. @@ -186,3 +257,119 @@ def add_table(self, tel_id, input_table): self._interpolators[tel_id] = {} self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) + + +class FlatFieldInterpolator(Interpolator): + """ + Interpolator for flatfield data + """ + + telescope_data_group = "dl1/calibration/gain" # TBD + required_columns = frozenset(["start_time", "end_time", "gain"]) + expected_units = {"gain": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate flatfield data for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + ffield : array [float] + interpolated flatfield data + """ + + self._check_interpolators(tel_id) + + ffield = self._interpolators[tel_id]["gain"](time) + return ffield + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "gain" + for the flatfield data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] + gain = input_table["gain"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["gain"] = ChunkFunction( + start_time, end_time, gain, **self.interp_options + ) + + +class PedestalInterpolator(Interpolator): + """ + Interpolator for Pedestal data + """ + + telescope_data_group = "dl1/calibration/pedestal" # TBD + required_columns = frozenset(["start_time", "end_time", "pedestal"]) + expected_units = {"pedestal": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate pedestal or gain for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + pedestal : array [float] + interpolated pedestal values + """ + + self._check_interpolators(tel_id) + + pedestal = self._interpolators[tel_id]["pedestal"](time) + return pedestal + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "pedestal" + for the pedestal data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] + pedestal = input_table["pedestal"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["pedestal"] = ChunkFunction( + start_time, end_time, pedestal, **self.interp_options + ) From 12ae9fc492e0659a2c6138d2f61f869b5503e3e8 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 08:51:19 +0200 Subject: [PATCH 109/221] added stats extractor parent component added PlainExtractor based on numpy and scipy functions --- src/ctapipe/calib/camera/extractor.py | 86 +++++++++++++++++++++++++++ src/ctapipe/containers.py | 3 + 2 files changed, 89 insertions(+) create mode 100644 src/ctapipe/calib/camera/extractor.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py new file mode 100644 index 00000000000..0140ed5bcb4 --- /dev/null +++ b/src/ctapipe/calib/camera/extractor.py @@ -0,0 +1,86 @@ +""" +Extraction algorithms to compute the statistics from a sequence of images +""" + +__all__ = [ + "StatisticsExtractor", + "PlainExtractor", +] + + +from abc import abstractmethod + +import numpy as np +import scipy.stats +from traitlets import Int + +from ctapipe.core import TelescopeComponent +from ctapipe.containers import StatisticsContainer + + +class StatisticsExtractor(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base component to handle the extraction of the statistics + from a sequence of charges and pulse times (images). + + Parameters + ---------- + kwargs + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def __call__(self, images, trigger_times) -> list: + """ + Call the relevant functions to extract the statistics + for the particular extractor. + + Parameters + ---------- + images : ndarray + images stored in a numpy array of shape + (n_images, n_channels, n_pix). + trigger_times : ndarray + images stored in a numpy array of shape + (n_images, ) + + Returns + ------- + List StatisticsContainer: + List of extracted statistics and validity ranges + """ + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(img, time)) + + return stats_list + + def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=np.nanmean(images, axis=0), + median=np.nanmedian(images, axis=0), + std=np.nanstd(images, axis=0), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 311142311a3..ec4f75f97ef 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -445,9 +445,12 @@ class StatisticsContainer(Container): class ImageStatisticsContainer(Container): """Store descriptive image statistics""" + validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") From cfe832863496aea421329202943fe618eb912d4d Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 15:51:47 +0200 Subject: [PATCH 110/221] added stats extractor based on sigma clipping --- src/ctapipe/calib/camera/extractor.py | 67 ++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0140ed5bcb4..654be103f8b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,6 +5,7 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "SigmaClippingExtractor", ] @@ -12,10 +13,14 @@ import numpy as np import scipy.stats -from traitlets import Int +from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core.traits import ( + Int, + List, +) class StatisticsExtractor(TelescopeComponent): @@ -84,3 +89,63 @@ def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: skewness=scipy.stats.skew(images, axis=0), kurtosis=scipy.stats.kurtosis(images, axis=0), ) + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(img, time)) + + return stats_list + + def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + + # mean, median, and std over the sample per pixel + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + std=pixel_std.filled(np.nan), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) + From 4d02301d0780d782426b47e4562687f940593415 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Tue, 30 Apr 2024 16:34:57 +0200 Subject: [PATCH 111/221] added cut of outliers restructured the stats containers --- src/ctapipe/calib/camera/extractor.py | 139 +++++++++++++++------ src/ctapipe/containers.py | 29 ++--- src/ctapipe/image/tests/test_statistics.py | 2 +- 3 files changed, 110 insertions(+), 60 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 654be103f8b..6e7ca6aa634 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -24,6 +24,17 @@ class StatisticsExtractor(TelescopeComponent): + + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics @@ -36,19 +47,18 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, images, trigger_times) -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: """ Call the relevant functions to extract the statistics for the particular extractor. Parameters ---------- - images : ndarray - images stored in a numpy array of shape + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). - trigger_times : ndarray - images stored in a numpy array of shape - (n_images, ) + col_name : string + column name in the dl1 table Returns ------- @@ -62,42 +72,60 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - sample_size = Int(2500, help="sample size").tag(config=True) + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: - def __call__(self, dl1_table, col_name="image") -> list: - # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), - mean=np.nanmean(images, axis=0), - median=np.nanmedian(images, axis=0), - std=np.nanstd(images, axis=0), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using astropy's sigma clipping functions """ - sample_size = Int(2500, help="sample size").tag(config=True) - sigma_clipping_max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", @@ -107,8 +135,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - - def __call__(self, dl1_table, col_name="image") -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) @@ -116,17 +143,26 @@ def __call__(self, dl1_table, col_name="image") -> list: # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, + masked_images, + sigma=max_sigma, maxiters=self.sigma_clipping_iterations, cenfunc="mean", axis=0, @@ -137,15 +173,42 @@ def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContaine pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), std=pixel_std.filled(np.nan), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + std_outliers=image_std_outliers.filled(True), ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index ec4f75f97ef..2e38d122285 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -422,35 +422,22 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a chunk of images""" - - n_events = Field(-1, "number of events used for the extraction of the statistics") - mean = Field( - None, - "mean of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) - median = Field( - None, - "median of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) - std = Field( - None, - "standard deviation of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) + """Store descriptive statistics of a sequence of images""" +validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") + mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") + median_outliers = Field(np.float32(nan), "median intensity") + std = Field(np.float32(nan), "standard deviation of intensity") + std_outliers = Field(np.float32(nan), "standard deviation intensity") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" - validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 4403e05ca0a..23806705787 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer + from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From c16b044d03f901f98bfe91274385bf33ec69a4df Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:10:59 +0200 Subject: [PATCH 112/221] update docs --- src/ctapipe/calib/camera/extractor.py | 4 +++- src/ctapipe/containers.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 6e7ca6aa634..56b8a7f2219 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,7 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) + sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -57,6 +57,8 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> dl1_table : ndarray dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). + masked_pixels_of_sample : ndarray + boolean array of masked pixels that are not available for processing col_name : string column name in the dl1 table diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 2e38d122285..9a4678401d8 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -424,13 +424,13 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" -validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") - mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") - median_outliers = Field(np.float32(nan), "median intensity") - std = Field(np.float32(nan), "standard deviation of intensity") - std_outliers = Field(np.float32(nan), "standard deviation intensity") + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + median = Field(np.float32(nan), "Channel-wise and pixel-wise median") + median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") + std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") + std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 3a82cec4dffaea07c29029ab97e98a3b94793d5b Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:12:33 +0200 Subject: [PATCH 113/221] formatted with black --- src/ctapipe/calib/camera/extractor.py | 87 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 56b8a7f2219..907f22923d2 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,10 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) + sample_size = Int( + 2500, + help="Size of the sample used for the calculation of the statistical values", + ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -47,7 +50,9 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -68,31 +73,41 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> List of extracted statistics and validity ranges """ + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using numpy and scipy functions """ - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -137,25 +152,34 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median of the event images image_median = np.ma.median(masked_images, axis=-1) @@ -194,15 +218,21 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = ( - np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], - image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = ( - np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) return StatisticsContainer( validity_start=times[0], @@ -213,4 +243,3 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S std=pixel_std.filled(np.nan), std_outliers=image_std_outliers.filled(True), ) - From 5646b2bae0f3e8223f079acee9ab45c60eef5573 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:29:10 +0200 Subject: [PATCH 114/221] added pass for __call__ function --- src/ctapipe/calib/camera/extractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 907f22923d2..e4ed0342e1b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -72,6 +72,7 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ + pass class PlainExtractor(StatisticsExtractor): From 487da6a311793b171fb8c28e807fddcea5b2757e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 7 May 2024 09:30:29 +0200 Subject: [PATCH 115/221] Small commit for prototyping --- src/ctapipe/calib/camera/extractor.py | 46 ++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index e4ed0342e1b..718aa48494b 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -6,6 +6,7 @@ "StatisticsExtractor", "PlainExtractor", "SigmaClippingExtractor", + "StarExtractor", ] @@ -14,6 +15,8 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats +from astropy.coordinates import EarthLocation, SkyCoord, Angle +from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -21,6 +24,7 @@ Int, List, ) +from ctapipe.coordinates import EngineeringCameraFrame class StatisticsExtractor(TelescopeComponent): @@ -138,9 +142,49 @@ def _plain_extraction( ) +class StarExtractor(StatisticsExtractor): + """ + Extracts pointing information from a series of variance images + using the startracker functions + """ + + min_star_magnitude = Float( + 0.1, + help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + ).tag(config=True) + + def __init__(): + + def __call__( + self, variance_table, initial_pointing, PSF_model + ): + + def _stars_in_FOV( + self, pointing + ): + + stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] + + for star in stars: + + star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') + star_coords = star_coords.transform_to(camera_frame) + central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( + + def _star_extraction( + self, + ): + camera_frame = EngineeringCameraFrame( + telescope_pointing=current_pointing, + focal_length=self.focal_length, + obstime=time.utc, + + + + class SigmaClippingExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extracts the statistics from a sequence of images using astropy's sigma clipping functions """ From 8b5b55e64897d22b216a7d5bfd5ca1e94255e41a Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 10 May 2024 09:05:01 +0200 Subject: [PATCH 116/221] Removed unneeded functions --- src/ctapipe/calib/camera/extractor.py | 28 +++------------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 718aa48494b..eb245447527 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,8 +5,8 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "StarVarianceExtractor", "SigmaClippingExtractor", - "StarExtractor", ] @@ -15,8 +15,6 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats -from astropy.coordinates import EarthLocation, SkyCoord, Angle -from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -142,7 +140,7 @@ def _plain_extraction( ) -class StarExtractor(StatisticsExtractor): +class StarVarianceExtractor(StatisticsExtractor): """ Extracts pointing information from a series of variance images using the startracker functions @@ -156,29 +154,9 @@ class StarExtractor(StatisticsExtractor): def __init__(): def __call__( - self, variance_table, initial_pointing, PSF_model + self, variance_table, trigger_table, initial_pointing, PSF_model ): - def _stars_in_FOV( - self, pointing - ): - - stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] - - for star in stars: - - star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') - star_coords = star_coords.transform_to(camera_frame) - central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( - - def _star_extraction( - self, - ): - camera_frame = EngineeringCameraFrame( - telescope_pointing=current_pointing, - focal_length=self.focal_length, - obstime=time.utc, - From 6e918dfec767b528ddeb61b938c101d37359bacd Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:41:44 +0200 Subject: [PATCH 117/221] added unit tests --- .../calib/camera/tests/test_extractors.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..5e5f7a617fc --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,51 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import astropy.units as u +from astropy.table import QTable +import numpy as np + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +def test_extractors(example_subarray): + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False + +def test_check_outliers(example_subarray): + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:,0,120] = 120.0 + flatfield_dl1_data[:,1,67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + #check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] == True + assert sigmaclipping_stats_list[0].median_outliers[1][67] == True + assert sigmaclipping_stats_list[1].median_outliers[0][120] == True + assert sigmaclipping_stats_list[1].median_outliers[1][67] == True From 169ca3761802c0067220529a6e79d2a142ee5913 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:50:48 +0200 Subject: [PATCH 118/221] added changelog --- docs/changes/2554.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/2554.feature.rst diff --git a/docs/changes/2554.feature.rst b/docs/changes/2554.feature.rst new file mode 100644 index 00000000000..2e6a6356b3a --- /dev/null +++ b/docs/changes/2554.feature.rst @@ -0,0 +1 @@ +Add API to extract the statistics from a sequence of images. From c78d1dc4e5e965d67ede89185eaf3b461ca6f303 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 13:54:45 +0200 Subject: [PATCH 119/221] fix lint --- src/ctapipe/calib/camera/extractor.py | 9 --------- src/ctapipe/calib/camera/tests/test_extractors.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index eb245447527..c8a1b3158ce 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -121,9 +121,6 @@ def _plain_extraction( # std over the sample per pixel pixel_std = np.ma.std(masked_images, axis=0) - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - # outliers from median image_median_outliers = np.logical_or( pixel_median < self.image_median_cut_outliers[0], @@ -204,9 +201,6 @@ def _sigmaclipping_extraction( # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - # median of the event images - image_median = np.ma.median(masked_images, axis=-1) - # mean, median, and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( @@ -224,9 +218,6 @@ def _sigmaclipping_extraction( unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - # only warn for values discard in the sigma clipping, not those from before - outliers = unused_values & (~masked_images.mask) - # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 5e5f7a617fc..19b04c017fe 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,7 +22,7 @@ def test_extractors(example_subarray): plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False From ff6ab1ddd44f027c84c658e0324e81f5690d3c1f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 23 May 2024 11:37:46 +0200 Subject: [PATCH 120/221] I altered the class variables to th evariance statistics extractor --- src/ctapipe/calib/camera/extractor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index c8a1b3158ce..70f4a2eb561 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -143,20 +143,23 @@ class StarVarianceExtractor(StatisticsExtractor): using the startracker functions """ - min_star_magnitude = Float( - 0.1, - help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) def __init__(): def __call__( - self, variance_table, trigger_table, initial_pointing, PSF_model + self, variance_table ): - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images From f44f4d2b79edd7cd3fca60997efa471932a8bf25 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 24 May 2024 13:38:36 +0200 Subject: [PATCH 121/221] added a container for mean variance images and fixed docustring --- src/ctapipe/calib/camera/extractor.py | 43 +++++++++++++++++++++++++-- src/ctapipe/containers.py | 9 +++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 70f4a2eb561..3cc51de40cb 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -139,8 +139,9 @@ def _plain_extraction( class StarVarianceExtractor(StatisticsExtractor): """ - Extracts pointing information from a series of variance images - using the startracker functions + Generating average variance images from a set + of variance images for the startracker + pointing calibration """ sigma_clipping_max_sigma = Int( @@ -158,7 +159,43 @@ def __call__( self, variance_table ): - + image_chunks = ( + variance_table["image"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["image"].data), self.sample_size) + ) + + time_chunks = ( + variance_table["trigger_times"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) + ) + + stats_list = [] + + for images, times in zip(image_chunks, time_chunks): + + stats_list.append( + self._sigmaclipping_extraction(images, times) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times + )->StatisticsContainer: + + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan) + ) class SigmaClippingExtractor(StatisticsExtractor): """ diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 9a4678401d8..b364aefa0cc 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -432,6 +433,13 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") +class VarianceStatisticsContainer(Container): + """Store descriptive statistics of a sequence of variance images""" + + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + class ImageStatisticsContainer(Container): """Store descriptive image statistics""" @@ -442,7 +450,6 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") - class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From 2200aece3d5e834c1a7ed153ba9e933edafe0dee Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 27 May 2024 15:20:53 +0200 Subject: [PATCH 122/221] I changed the container type for the StarVarianceExtractor --- src/ctapipe/calib/camera/extractor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 3cc51de40cb..ffcd561aa87 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -180,7 +180,7 @@ def __call__( def _sigmaclipping_extraction( self, images, times - )->StatisticsContainer: + )->VarianceStatisticsContainer: pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( images, @@ -190,11 +190,10 @@ def _sigmaclipping_extraction( axis=0, ) - return StatisticsContainer( + return VarianceStatisticsContainer( validity_start=times[0], validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan) + mean=pixel_mean.filled(np.nan) ) class SigmaClippingExtractor(StatisticsExtractor): From c4d6809798368718fde73277912a4bc084c54abc Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:42:35 +0200 Subject: [PATCH 123/221] fix pylint Remove StarVarianceExtractor since is functionality is featured in the existing Extractors --- src/ctapipe/calib/camera/extractor.py | 89 ++++--------------- .../calib/camera/tests/test_extractors.py | 64 +++++++------ src/ctapipe/containers.py | 7 -- 3 files changed, 53 insertions(+), 107 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index ffcd561aa87..9e9e9947462 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,15 +5,12 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", - "StarVarianceExtractor", "SigmaClippingExtractor", ] - from abc import abstractmethod import numpy as np -import scipy.stats from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent @@ -22,10 +19,9 @@ Int, List, ) -from ctapipe.coordinates import EngineeringCameraFrame - class StatisticsExtractor(TelescopeComponent): + """Base StatisticsExtractor component""" sample_size = Int( 2500, @@ -33,11 +29,13 @@ class StatisticsExtractor(TelescopeComponent): ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", + help="""Interval of accepted image values \\ + (fraction with respect to camera median value)""", ).tag(config=True) image_std_cut_outliers = List( [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", + help="""Interval (number of std) of accepted image standard deviation \\ + around camera median value""", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): @@ -74,7 +72,6 @@ def __call__( List StatisticsContainer: List of extracted statistics and validity ranges """ - pass class PlainExtractor(StatisticsExtractor): @@ -136,66 +133,6 @@ def _plain_extraction( std=pixel_std.filled(np.nan), ) - -class StarVarianceExtractor(StatisticsExtractor): - """ - Generating average variance images from a set - of variance images for the startracker - pointing calibration - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __init__(): - - def __call__( - self, variance_table - ): - - image_chunks = ( - variance_table["image"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["image"].data), self.sample_size) - ) - - time_chunks = ( - variance_table["trigger_times"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) - ) - - stats_list = [] - - for images, times in zip(image_chunks, time_chunks): - - stats_list.append( - self._sigmaclipping_extraction(images, times) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times - )->VarianceStatisticsContainer: - - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - return VarianceStatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan) - ) - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images @@ -273,18 +210,26 @@ def _sigmaclipping_extraction( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], ) return StatisticsContainer( diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 19b04c017fe..89c375387e3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,7 +2,6 @@ Tests for StatisticsExtractor and related functions """ -import astropy.units as u from astropy.table import QTable import numpy as np @@ -10,42 +9,51 @@ def test_extractors(example_subarray): + """test basic functionality of the StatisticsExtractors""" + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) - + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False - + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + + def test_check_outliers(example_subarray): - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + """test detection ability of outliers""" + + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers - flatfield_dl1_data[:,0,120] = 120.0 - flatfield_dl1_data[:,1,67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - #check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] == True - assert sigmaclipping_stats_list[0].median_outliers[1][67] == True - assert sigmaclipping_stats_list[1].median_outliers[0][120] == True - assert sigmaclipping_stats_list[1].median_outliers[1][67] == True + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] is True + assert sigmaclipping_stats_list[0].median_outliers[1][67] is True + assert sigmaclipping_stats_list[1].median_outliers[0][120] is True + assert sigmaclipping_stats_list[1].median_outliers[1][67] is True diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index b364aefa0cc..bff71facca4 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,7 +57,6 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", - "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -433,12 +432,6 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") -class VarianceStatisticsContainer(Container): - """Store descriptive statistics of a sequence of variance images""" - - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From 388aae69a859dee3299a9ee2b0c786725ef4ce7d Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:45:29 +0200 Subject: [PATCH 124/221] change __call__() to _extract() --- src/ctapipe/calib/camera/extractor.py | 6 +++--- src/ctapipe/calib/camera/tests/test_extractors.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 9e9e9947462..eaff6c714c5 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -80,7 +80,7 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: @@ -148,7 +148,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 89c375387e3..d5f082762ed 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,8 +22,8 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -50,7 +50,7 @@ def test_check_outliers(example_subarray): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From 07f4bef83db26aa65d3972e2a3b5f4e1d0b8e448 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:59:36 +0200 Subject: [PATCH 125/221] minor renaming --- src/ctapipe/calib/camera/extractor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index eaff6c714c5..0c23caebb00 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -60,7 +60,7 @@ def _extract( Parameters ---------- dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape + dl1 table with images and timestamps stored in a numpy array of shape (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing @@ -139,11 +139,11 @@ class SigmaClippingExtractor(StatisticsExtractor): using astropy's sigma clipping functions """ - sigma_clipping_max_sigma = Int( + max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", ).tag(config=True) - sigma_clipping_iterations = Int( + iterations = Int( default_value=5, help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) @@ -178,11 +178,10 @@ def _sigmaclipping_extraction( masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, + sigma=self.max_sigma, + maxiters=self.iterations, cenfunc="mean", axis=0, ) @@ -192,7 +191,7 @@ def _sigmaclipping_extraction( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From a3782ae93134f810b36123e885c7463d991ceb1c Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:15:53 +0200 Subject: [PATCH 126/221] use pytest.fixture for Extractors --- .../calib/camera/tests/test_extractors.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d5f082762ed..d363ee24ff0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -7,14 +7,21 @@ from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor +@pytest.fixture + def test_plainextractor(example_subarray): + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -def test_extractors(example_subarray): +@pytest.fixture + def test_sigmaclippingextractor(example_subarray): + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" - plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) @@ -22,8 +29,10 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + dl1_table=flatfield_dl1_table + ) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -38,19 +47,18 @@ def test_extractors(example_subarray): assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False -def test_check_outliers(example_subarray): +def test_check_outliers(test_sigmaclippingextractor): """test detection ability of outliers""" - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract( + dl1_table=flatfield_dl1_table + ) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From ad475606e332a5b4593cce7ccc61611f85a9aad3 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:59:57 +0200 Subject: [PATCH 127/221] reduce duplicated code of the call function --- src/ctapipe/calib/camera/extractor.py | 52 ++++++------------- .../calib/camera/tests/test_extractors.py | 30 ++++++----- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 0c23caebb00..d060d620508 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -49,8 +49,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - @abstractmethod - def _extract( + def __call__( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -73,17 +72,6 @@ def _extract( List of extracted statistics and validity ranges """ - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] @@ -98,11 +86,23 @@ def _extract( stats_list = [] for images, times in zip(image_chunks, time_chunks): stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) + self._extract(images, times, masked_pixels_of_sample) ) return stats_list - def _plain_extraction( + @abstractmethod + def _extract( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + pass + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def _extract( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: @@ -149,28 +149,6 @@ class SigmaClippingExtractor(StatisticsExtractor): ).tag(config=True) def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d363ee24ff0..06107a6e7b7 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -4,20 +4,22 @@ from astropy.table import QTable import numpy as np - +import pytest from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor -@pytest.fixture - def test_plainextractor(example_subarray): - return PlainExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -@pytest.fixture - def test_sigmaclippingextractor(example_subarray): - return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -29,8 +31,8 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) @@ -56,7 +58,7 @@ def test_check_outliers(test_sigmaclippingextractor): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract( + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) From 8f498024461e1b3b8ebe1edfe760f256f9c10828 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 128/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 223 ++++++++++++++++++++++++- src/ctapipe/image/psf_model.py | 80 +++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index baf3d2f1057..81cacedc432 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,24 +2,33 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Integer, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource -__all__ = ["CameraCalibrator"] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -47,6 +56,218 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) + + @abstractmethod + def __call__(self, data_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated + tel_id : int + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties + """ + + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) + + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): + """ + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + pass + + +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + PSFModel_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="PlainExtractor", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + self.psf = PSFModel.from_name( + self.PSFModel_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, url, tel_id): + if self._check_req_data(url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + with EventSource(url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + def _calibrate_varimages(self, varimages, gain): + pass + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..7af526e6c21 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,80 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + +from ctapipe.core import TelescopeComponent + + +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): + """ + Base Component to describe image distortion due to the optics of the different cameras. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) From 9d86e9a7dae56b19ae93fa022b4eb761ff3a90ab Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 129/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 81cacedc432..cd7ad324e1f 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -266,6 +266,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From cd122a86b18634c7f160ab3965fa0ab065945b83 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 130/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index cd7ad324e1f..0c48113e4ae 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -111,21 +111,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -208,7 +220,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -224,7 +236,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -264,7 +276,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From 12cc7663e4cabf9d4c6a491bc20d770be43b03ce Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 131/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 0c48113e4ae..f921b2d97fb 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -276,9 +276,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From 20de205b9f44ac7bb83aec85c89f9454b12d2b9b Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 12:02:41 +0200 Subject: [PATCH 132/221] edit description of StatisticsContainer --- src/ctapipe/containers.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index bff71facca4..40bd34b088d 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -424,13 +424,33 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") - median = Field(np.float32(nan), "Channel-wise and pixel-wise median") - median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") - std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") - std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") + extraction_start = Field(np.float32(nan), "start of the extraction sequence") + extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + mean = Field( + None, + "mean of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median = Field( + None, + "median of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median_outliers = Field( + None, + "outliers from the median distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) + std = Field( + None, + "standard deviation of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + std_outliers = Field( + None, + "outliers from the standard deviation distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) class ImageStatisticsContainer(Container): @@ -443,6 +463,7 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") + class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From c4936d72e4329536ce8bb63eb13c31756803766f Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 13:12:54 +0200 Subject: [PATCH 133/221] added feature to shift the extraction sequence allow overlapping extraction sequences --- src/ctapipe/calib/camera/extractor.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index d060d620508..f4ddb03d4bd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,7 +50,11 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" + self, + dl1_table, + masked_pixels_of_sample=None, + sample_shift=None, + col_name="image", ) -> list: """ Call the relevant functions to extract the statistics @@ -63,6 +67,8 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing + sample_shift : int + number of samples to shift the extraction sequence col_name : string column name in the dl1 table @@ -72,14 +78,19 @@ def __call__( List of extracted statistics and validity ranges """ + # If no sample_shift is provided, the sample_shift is set to self.sample_size + # meaning that the samples are not overlapping. + if sample_shift is None: + sample_shift = self.sample_size + # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) + for i in range(0, len(dl1_table[col_name].data), sample_shift) ) time_chunks = ( dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) + for i in range(0, len(dl1_table["time"]), sample_shift) ) # Calculate the statistics from a sequence of images @@ -169,7 +180,9 @@ def _extract( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > ( + self.max_sigma * pixel_std + ) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From f7fefd178247b544424dbc36012efb709bb5b123 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 14:23:53 +0200 Subject: [PATCH 134/221] fix boundary case for the last chunk renaming to chunk(s) and chunk_size and _shift added test for chunk_shift and boundary case --- src/ctapipe/calib/camera/extractor.py | 72 ++++++++++--------- .../calib/camera/tests/test_extractors.py | 21 +++++- src/ctapipe/containers.py | 6 +- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index f4ddb03d4bd..86d9c1345fd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,5 +1,5 @@ """ -Extraction algorithms to compute the statistics from a sequence of images +Extraction algorithms to compute the statistics from a chunk of images """ __all__ = [ @@ -23,9 +23,9 @@ class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" - sample_size = Int( + chunk_size = Int( 2500, - help="Size of the sample used for the calculation of the statistical values", + help="Size of the chunk used for the calculation of the statistical values", ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], @@ -41,7 +41,7 @@ class StatisticsExtractor(TelescopeComponent): def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). + from a chunk of charges and pulse times (images). Parameters ---------- @@ -53,7 +53,7 @@ def __call__( self, dl1_table, masked_pixels_of_sample=None, - sample_shift=None, + chunk_shift=None, col_name="image", ) -> list: """ @@ -67,38 +67,44 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing - sample_shift : int - number of samples to shift the extraction sequence + chunk_shift : int + number of samples to shift the extraction chunk col_name : string column name in the dl1 table Returns ------- List StatisticsContainer: - List of extracted statistics and validity ranges + List of extracted statistics and extraction chunks """ - # If no sample_shift is provided, the sample_shift is set to self.sample_size - # meaning that the samples are not overlapping. - if sample_shift is None: - sample_shift = self.sample_size - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), sample_shift) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), sample_shift) - ) - - # Calculate the statistics from a sequence of images + # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size + # meaning that the extraction chunks are not overlapping. + if chunk_shift is None: + chunk_shift = self.chunk_size + + # Function to split table data into appropriated chunks + def _get_chunks(col_name): + return [ + ( + dl1_table[col_name].data[i : i + self.chunk_size] + if i + self.chunk_size <= len(dl1_table[col_name]) + else dl1_table[col_name].data[ + len(dl1_table[col_name].data) + - self.chunk_size : len(dl1_table[col_name].data) + ] + ) + for i in range(0, len(dl1_table[col_name].data), chunk_shift) + ] + + # Get the chunks for the timestamps and selected column name + time_chunks = _get_chunks("time") + image_chunks = _get_chunks(col_name) + + # Calculate the statistics from a chunk of images stats_list = [] for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._extract(images, times, masked_pixels_of_sample) - ) + stats_list.append(self._extract(images, times, masked_pixels_of_sample)) return stats_list @abstractmethod @@ -109,7 +115,7 @@ def _extract( class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extractor the statistics from a chunk of images using numpy and scipy functions """ @@ -136,8 +142,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), @@ -146,7 +152,7 @@ def _extract( class SigmaClippingExtractor(StatisticsExtractor): """ - Extracts the statistics from a sequence of images + Extracts the statistics from a chunk of images using astropy's sigma clipping functions """ @@ -223,8 +229,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 06107a6e7b7..40efd4f2fc3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -11,14 +11,14 @@ def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" return PlainExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): @@ -67,3 +67,20 @@ def test_check_outliers(test_sigmaclippingextractor): assert sigmaclipping_stats_list[0].median_outliers[1][67] is True assert sigmaclipping_stats_list[1].median_outliers[0][120] is True assert sigmaclipping_stats_list[1].median_outliers[1][67] is True + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, + chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 + diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 40bd34b088d..a937f49d337 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -422,10 +422,10 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a sequence of images""" + """Store descriptive statistics of a chunk of images""" - extraction_start = Field(np.float32(nan), "start of the extraction sequence") - extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + extraction_start = Field(np.float32(nan), "start of the extraction chunk") + extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") mean = Field( None, "mean of a pixel-wise quantity for each channel" From 56118b879c89c23cb4e65ba055e77030d1629b37 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:13:27 +0200 Subject: [PATCH 135/221] fix tests --- .../calib/camera/tests/test_extractors.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 40efd4f2fc3..a83c93fd1c0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,24 +2,24 @@ Tests for StatisticsExtractor and related functions """ -from astropy.table import QTable import numpy as np import pytest +from astropy.table import QTable + from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + @pytest.fixture(name="test_plainextractor") def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" - return PlainExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" - return SigmaClippingExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -36,17 +36,17 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): dl1_table=flatfield_dl1_table ) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) def test_check_outliers(test_sigmaclippingextractor): @@ -63,11 +63,11 @@ def test_check_outliers(test_sigmaclippingextractor): ) # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] is True - assert sigmaclipping_stats_list[0].median_outliers[1][67] is True - assert sigmaclipping_stats_list[1].median_outliers[0][120] is True - assert sigmaclipping_stats_list[1].median_outliers[1][67] is True - + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + def test_check_chunk_shift(test_sigmaclippingextractor): """test the chunk shift option and the boundary case for the last chunk""" @@ -77,10 +77,8 @@ def test_check_chunk_shift(test_sigmaclippingextractor): # insert outliers flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, - chunk_shift=2000 + dl1_table=flatfield_dl1_table, chunk_shift=2000 ) # check if three chunks are used for the extraction assert len(sigmaclipping_stats_list) == 3 - From e68c5ddacaca67660e168d0ba4b09044b3805116 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:26:03 +0200 Subject: [PATCH 136/221] fix ruff --- src/ctapipe/calib/camera/extractor.py | 40 +++++++++------------- src/ctapipe/image/tests/test_statistics.py | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 86d9c1345fd..4c8f49d1f38 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -13,13 +13,14 @@ import numpy as np from astropy.stats import sigma_clipped_stats -from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Int, List, ) + class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" @@ -90,8 +91,9 @@ def _get_chunks(col_name): dl1_table[col_name].data[i : i + self.chunk_size] if i + self.chunk_size <= len(dl1_table[col_name]) else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - - self.chunk_size : len(dl1_table[col_name].data) + len(dl1_table[col_name].data) - self.chunk_size : len( + dl1_table[col_name].data + ) ] ) for i in range(0, len(dl1_table[col_name].data), chunk_shift) @@ -108,21 +110,17 @@ def _get_chunks(col_name): return stats_list @abstractmethod - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: pass + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a chunk of images using numpy and scipy functions """ - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -150,6 +148,7 @@ def _extract( std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a chunk of images @@ -165,10 +164,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -206,25 +202,21 @@ def _extract( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], ) diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 23806705787..4403e05ca0a 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer + from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From 057baf6a7125ac5680e031bf53ff796f70662ab2 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 137/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index f921b2d97fb..360103982a8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -26,7 +26,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -270,6 +270,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -294,7 +299,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From d5eda7836fdb5cdf3bfb3e14e27b59ec4cf8e224 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 138/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 227 +++++++++++++++++-------- 1 file changed, 160 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 360103982a8..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,13 +2,17 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ + from abc import abstractmethod from functools import cache +import pathlib import astropy.units as u +from astropy.table import Table +import pickle + import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -20,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -28,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -72,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -95,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -235,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -281,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From 53d8e9869978243e8084742bea90c69c23cb82b2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 139/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From f982b363f5d0c99deaa4911cc008fbeb705064cb Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 08:51:19 +0200 Subject: [PATCH 140/221] added stats extractor parent component added PlainExtractor based on numpy and scipy functions --- src/ctapipe/calib/camera/extractor.py | 221 +++++--------------------- src/ctapipe/containers.py | 3 + 2 files changed, 42 insertions(+), 182 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 4c8f49d1f38..1c138892a47 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,48 +1,29 @@ """ -Extraction algorithms to compute the statistics from a chunk of images +Extraction algorithms to compute the statistics from a sequence of images """ __all__ = [ "StatisticsExtractor", "PlainExtractor", - "SigmaClippingExtractor", ] + from abc import abstractmethod import numpy as np -from astropy.stats import sigma_clipped_stats +import scipy.stats +from traitlets import Int -from ctapipe.containers import StatisticsContainer from ctapipe.core import TelescopeComponent -from ctapipe.core.traits import ( - Int, - List, -) +from ctapipe.containers import StatisticsContainer class StatisticsExtractor(TelescopeComponent): - """Base StatisticsExtractor component""" - - chunk_size = Int( - 2500, - help="Size of the chunk used for the calculation of the statistical values", - ).tag(config=True) - image_median_cut_outliers = List( - [-0.3, 0.3], - help="""Interval of accepted image values \\ - (fraction with respect to camera median value)""", - ).tag(config=True) - image_std_cut_outliers = List( - [-3, 3], - help="""Interval (number of std) of accepted image standard deviation \\ - around camera median value""", - ).tag(config=True) - def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a chunk of charges and pulse times (images). + from a sequence of charges and pulse times (images). +>>>>>>> 58d868c8 (added stats extractor parent component) Parameters ---------- @@ -50,182 +31,58 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__( - self, - dl1_table, - masked_pixels_of_sample=None, - chunk_shift=None, - col_name="image", - ) -> list: + @abstractmethod + def __call__(self, images, trigger_times) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. Parameters ---------- - dl1_table : ndarray - dl1 table with images and timestamps stored in a numpy array of shape + images : ndarray + images stored in a numpy array of shape (n_images, n_channels, n_pix). - masked_pixels_of_sample : ndarray - boolean array of masked pixels that are not available for processing - chunk_shift : int - number of samples to shift the extraction chunk - col_name : string - column name in the dl1 table + trigger_times : ndarray + images stored in a numpy array of shape + (n_images, ) Returns ------- List StatisticsContainer: - List of extracted statistics and extraction chunks - """ - - # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size - # meaning that the extraction chunks are not overlapping. - if chunk_shift is None: - chunk_shift = self.chunk_size - - # Function to split table data into appropriated chunks - def _get_chunks(col_name): - return [ - ( - dl1_table[col_name].data[i : i + self.chunk_size] - if i + self.chunk_size <= len(dl1_table[col_name]) - else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - self.chunk_size : len( - dl1_table[col_name].data - ) - ] - ) - for i in range(0, len(dl1_table[col_name].data), chunk_shift) - ] - - # Get the chunks for the timestamps and selected column name - time_chunks = _get_chunks("time") - image_chunks = _get_chunks(col_name) - - # Calculate the statistics from a chunk of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append(self._extract(images, times, masked_pixels_of_sample)) - return stats_list - - @abstractmethod - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - pass + List of extracted statistics and validity ranges + """ class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a chunk of images + Extractor the statistics from a sequence of images using numpy and scipy functions """ - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median over the sample per pixel - pixel_median = np.ma.median(masked_images, axis=0) - - # mean over the sample per pixel - pixel_mean = np.ma.mean(masked_images, axis=0) - - # std over the sample per pixel - pixel_std = np.ma.std(masked_images, axis=0) + sample_size = Int(2500, help="sample size").tag(config=True) - # outliers from median - image_median_outliers = np.logical_or( - pixel_median < self.image_median_cut_outliers[0], - pixel_median > self.image_median_cut_outliers[1], - ) - - return StatisticsContainer( - extraction_start=times[0], - extraction_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - ) + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(img, time)) + + return stats_list -class SigmaClippingExtractor(StatisticsExtractor): - """ - Extracts the statistics from a chunk of images - using astropy's sigma clipping functions - """ - - max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # mean, median, and std over the sample per pixel - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - masked_images, - sigma=self.max_sigma, - maxiters=self.iterations, - cenfunc="mean", - axis=0, - ) - - # mask pixels without defined statistical values - pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) - pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) - pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - - unused_values = np.abs(masked_images - pixel_mean) > ( - self.max_sigma * pixel_std - ) - - # add outliers identified by sigma clipping for following operations - masked_images.mask |= unused_values - - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # median of the std over the camera - median_of_pixel_std = np.ma.median(pixel_std, axis=1) - - # std of the std over camera - std_of_pixel_std = np.ma.std(pixel_std, axis=1) - - # outliers from median - image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = np.logical_or( - image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[:, np.newaxis], - image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[:, np.newaxis], - ) - - # outliers from standard deviation - deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = np.logical_or( - deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object - * std_of_pixel_std[:, np.newaxis], - deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object - * std_of_pixel_std[:, np.newaxis], - ) - + def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: return StatisticsContainer( - extraction_start=times[0], - extraction_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - std_outliers=image_std_outliers.filled(True), + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=np.nanmean(images, axis=0), + median=np.nanmedian(images, axis=0), + std=np.nanstd(images, axis=0), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index a937f49d337..ee1a66f9332 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -456,9 +456,12 @@ class StatisticsContainer(Container): class ImageStatisticsContainer(Container): """Store descriptive image statistics""" + validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") From 9c46f555dc4115cbef9c9b0d6480f8dd15aca1e7 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Mon, 29 Apr 2024 15:51:47 +0200 Subject: [PATCH 141/221] added stats extractor based on sigma clipping --- src/ctapipe/calib/camera/extractor.py | 67 ++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 1c138892a47..2dab8fef552 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,6 +5,7 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "SigmaClippingExtractor", ] @@ -12,10 +13,14 @@ import numpy as np import scipy.stats -from traitlets import Int +from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core.traits import ( + Int, + List, +) class StatisticsExtractor(TelescopeComponent): @@ -86,3 +91,63 @@ def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: skewness=scipy.stats.skew(images, axis=0), kurtosis=scipy.stats.kurtosis(images, axis=0), ) + +class SigmaClippingExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using astropy's sigma clipping functions + """ + + sample_size = Int(2500, help="sample size").tag(config=True) + + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", + ).tag(config=True) + + + def __call__(self, dl1_table, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for img, time in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(img, time)) + + return stats_list + + def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + + # mean, median, and std over the sample per pixel + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + # mask pixels without defined statistical values + pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) + pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) + pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + + return StatisticsContainer( + validity_start=trigger_times[0], + validity_stop=trigger_times[-1], + max=np.max(images, axis=0), + min=np.min(images, axis=0), + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + std=pixel_std.filled(np.nan), + skewness=scipy.stats.skew(images, axis=0), + kurtosis=scipy.stats.kurtosis(images, axis=0), + ) + From 57879b1581d0709b007f3909b0f8c2ad19302fff Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Tue, 30 Apr 2024 16:34:57 +0200 Subject: [PATCH 142/221] added cut of outliers restructured the stats containers --- src/ctapipe/calib/camera/extractor.py | 139 +++++++++++++++------ src/ctapipe/containers.py | 40 ++---- src/ctapipe/image/tests/test_statistics.py | 2 +- 3 files changed, 110 insertions(+), 71 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 2dab8fef552..1553d13eca5 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -24,6 +24,17 @@ class StatisticsExtractor(TelescopeComponent): + + sample_size = Int(2500, help="sample size").tag(config=True) + image_median_cut_outliers = List( + [-0.3, 0.3], + help="Interval of accepted image values (fraction with respect to camera median value)", + ).tag(config=True) + image_std_cut_outliers = List( + [-3, 3], + help="Interval (number of std) of accepted image standard deviation around camera median value", + ).tag(config=True) + def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics @@ -37,19 +48,18 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, images, trigger_times) -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: """ Call the relevant functions to extract the statistics for the particular extractor. Parameters ---------- - images : ndarray - images stored in a numpy array of shape + dl1_table : ndarray + dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). - trigger_times : ndarray - images stored in a numpy array of shape - (n_images, ) + col_name : string + column name in the dl1 table Returns ------- @@ -64,42 +74,60 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - sample_size = Int(2500, help="sample size").tag(config=True) + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: - def __call__(self, dl1_table, col_name="image") -> list: - # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _plain_extraction(self, images, trigger_times) -> StatisticsContainer: + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median over the sample per pixel + pixel_median = np.ma.median(masked_images, axis=0) + + # mean over the sample per pixel + pixel_mean = np.ma.mean(masked_images, axis=0) + + # std over the sample per pixel + pixel_std = np.ma.std(masked_images, axis=0) + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # outliers from median + image_median_outliers = np.logical_or( + pixel_median < self.image_median_cut_outliers[0], + pixel_median > self.image_median_cut_outliers[1], + ) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), - mean=np.nanmean(images, axis=0), - median=np.nanmedian(images, axis=0), - std=np.nanstd(images, axis=0), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), + std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using astropy's sigma clipping functions """ - sample_size = Int(2500, help="sample size").tag(config=True) - sigma_clipping_max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", @@ -109,8 +137,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - - def __call__(self, dl1_table, col_name="image") -> list: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: # in python 3.12 itertools.batched can be used image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) @@ -118,17 +145,26 @@ def __call__(self, dl1_table, col_name="image") -> list: # Calculate the statistics from a sequence of images stats_list = [] - for img, time in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(img, time)) - + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) return stats_list - def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContainer: + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + + # ensure numpy array + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, + masked_images, + sigma=max_sigma, maxiters=self.sigma_clipping_iterations, cenfunc="mean", axis=0, @@ -139,15 +175,42 @@ def _sigmaclipping_extraction(self, images, trigger_times) -> StatisticsContaine pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) + + # add outliers identified by sigma clipping for following operations + masked_images.mask |= unused_values + + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + + # median of the std over the camera + median_of_pixel_std = np.ma.median(pixel_std, axis=1) + + # std of the std over camera + std_of_pixel_std = np.ma.std(pixel_std, axis=1) + + # outliers from median + image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + + # outliers from standard deviation + deviation = pixel_std - median_of_pixel_std[:, np.newaxis] + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + return StatisticsContainer( - validity_start=trigger_times[0], - validity_stop=trigger_times[-1], - max=np.max(images, axis=0), - min=np.min(images, axis=0), + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), + median_outliers=image_median_outliers.filled(True), std=pixel_std.filled(np.nan), - skewness=scipy.stats.skew(images, axis=0), - kurtosis=scipy.stats.kurtosis(images, axis=0), + std_outliers=image_std_outliers.filled(True), ) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index ee1a66f9332..2485bc1e532 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -422,46 +422,22 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a chunk of images""" - - extraction_start = Field(np.float32(nan), "start of the extraction chunk") - extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") - mean = Field( - None, - "mean of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) - median = Field( - None, - "median of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) - median_outliers = Field( - None, - "outliers from the median distribution of a pixel-wise quantity for each channel" - "Type: binary mask; Shape: (n_channels, n_pixel)", - ) - std = Field( - None, - "standard deviation of a pixel-wise quantity for each channel" - "Type: float; Shape: (n_channels, n_pixel)", - ) - std_outliers = Field( - None, - "outliers from the standard deviation distribution of a pixel-wise quantity for each channel" - "Type: binary mask; Shape: (n_channels, n_pixel)", - ) + """Store descriptive statistics of a sequence of images""" + validity_start = Field(np.float32(nan), "start") + validity_stop = Field(np.float32(nan), "stop") + mean = Field(np.float32(nan), "mean intensity") + median = Field(np.float32(nan), "median intensity") + median_outliers = Field(np.float32(nan), "median intensity") + std = Field(np.float32(nan), "standard deviation of intensity") + std_outliers = Field(np.float32(nan), "standard deviation intensity") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" - validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") max = Field(np.float32(nan), "value of pixel with maximum intensity") min = Field(np.float32(nan), "value of pixel with minimum intensity") mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") std = Field(np.float32(nan), "standard deviation of intensity") skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 4403e05ca0a..23806705787 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer + from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From fd7947903c4bcbce9f8518641194fa58e0e7ce79 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:10:59 +0200 Subject: [PATCH 143/221] update docs --- src/ctapipe/calib/camera/extractor.py | 4 +++- src/ctapipe/containers.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 1553d13eca5..3bab15b6865 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,7 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) + sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -58,6 +58,8 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> dl1_table : ndarray dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). + masked_pixels_of_sample : ndarray + boolean array of masked pixels that are not available for processing col_name : string column name in the dl1 table diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 2485bc1e532..9a4678401d8 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -424,13 +424,13 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start") - validity_stop = Field(np.float32(nan), "stop") - mean = Field(np.float32(nan), "mean intensity") - median = Field(np.float32(nan), "median intensity") - median_outliers = Field(np.float32(nan), "median intensity") - std = Field(np.float32(nan), "standard deviation of intensity") - std_outliers = Field(np.float32(nan), "standard deviation intensity") + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + median = Field(np.float32(nan), "Channel-wise and pixel-wise median") + median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") + std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") + std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From b886875574e25dce126ce0a97cfd20244f3e821a Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:12:33 +0200 Subject: [PATCH 144/221] formatted with black --- src/ctapipe/calib/camera/extractor.py | 87 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 3bab15b6865..9075eba616c 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -25,7 +25,10 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="Size of the sample used for the calculation of the statistical values").tag(config=True) + sample_size = Int( + 2500, + help="Size of the sample used for the calculation of the statistical values", + ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], help="Interval of accepted image values (fraction with respect to camera median value)", @@ -48,7 +51,9 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -70,31 +75,41 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> List of extracted statistics and validity ranges """ + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using numpy and scipy functions """ - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -139,25 +154,34 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median of the event images image_median = np.ma.median(masked_images, axis=-1) @@ -196,15 +220,21 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = ( - np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], - image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = ( - np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) return StatisticsContainer( validity_start=times[0], @@ -215,4 +245,3 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S std=pixel_std.filled(np.nan), std_outliers=image_std_outliers.filled(True), ) - From 10723a3246c370fa7c89cc5d031a4e34b5ab35b8 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Thu, 2 May 2024 10:29:10 +0200 Subject: [PATCH 145/221] added pass for __call__ function --- src/ctapipe/calib/camera/extractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 9075eba616c..c71785829c0 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -74,6 +74,7 @@ def __call__( List of extracted statistics and validity ranges """ + pass class PlainExtractor(StatisticsExtractor): From 9c467482063953b587413ba3045f595072c4b32f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 7 May 2024 09:30:29 +0200 Subject: [PATCH 146/221] Small commit for prototyping --- src/ctapipe/calib/camera/extractor.py | 46 ++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index c71785829c0..625a6b02ecb 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -6,6 +6,7 @@ "StatisticsExtractor", "PlainExtractor", "SigmaClippingExtractor", + "StarExtractor", ] @@ -14,6 +15,8 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats +from astropy.coordinates import EarthLocation, SkyCoord, Angle +from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -21,6 +24,7 @@ Int, List, ) +from ctapipe.coordinates import EngineeringCameraFrame class StatisticsExtractor(TelescopeComponent): @@ -140,9 +144,49 @@ def _plain_extraction( ) +class StarExtractor(StatisticsExtractor): + """ + Extracts pointing information from a series of variance images + using the startracker functions + """ + + min_star_magnitude = Float( + 0.1, + help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + ).tag(config=True) + + def __init__(): + + def __call__( + self, variance_table, initial_pointing, PSF_model + ): + + def _stars_in_FOV( + self, pointing + ): + + stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] + + for star in stars: + + star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') + star_coords = star_coords.transform_to(camera_frame) + central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( + + def _star_extraction( + self, + ): + camera_frame = EngineeringCameraFrame( + telescope_pointing=current_pointing, + focal_length=self.focal_length, + obstime=time.utc, + + + + class SigmaClippingExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extracts the statistics from a sequence of images using astropy's sigma clipping functions """ From a08b38ce5d34a48f63d6564be2b8413e72d51c1f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 10 May 2024 09:05:01 +0200 Subject: [PATCH 147/221] Removed unneeded functions --- src/ctapipe/calib/camera/extractor.py | 28 +++------------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 625a6b02ecb..45bbb27c3e1 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,8 +5,8 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", + "StarVarianceExtractor", "SigmaClippingExtractor", - "StarExtractor", ] @@ -15,8 +15,6 @@ import numpy as np import scipy.stats from astropy.stats import sigma_clipped_stats -from astropy.coordinates import EarthLocation, SkyCoord, Angle -from astroquery.vizier import Vizier from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer @@ -144,7 +142,7 @@ def _plain_extraction( ) -class StarExtractor(StatisticsExtractor): +class StarVarianceExtractor(StatisticsExtractor): """ Extracts pointing information from a series of variance images using the startracker functions @@ -158,29 +156,9 @@ class StarExtractor(StatisticsExtractor): def __init__(): def __call__( - self, variance_table, initial_pointing, PSF_model + self, variance_table, trigger_table, initial_pointing, PSF_model ): - def _stars_in_FOV( - self, pointing - ): - - stars = Vizier.query_region(pointing, radius=Angle(2.0, "deg"),catalog='NOMAD')[0] - - for star in stars: - - star_coords = SkyCoord(star['RAJ2000'], star['DEJ2000'], unit='deg', frame='icrs') - star_coords = star_coords.transform_to(camera_frame) - central_pixel = self.camera_geometry.transform_to(camera_frame).position_to_pix_index( - - def _star_extraction( - self, - ): - camera_frame = EngineeringCameraFrame( - telescope_pointing=current_pointing, - focal_length=self.focal_length, - obstime=time.utc, - From 4f063449d28fd6318faeeb45c6e195733fa5644b Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 11:41:44 +0200 Subject: [PATCH 148/221] added unit tests --- .../calib/camera/tests/test_extractors.py | 103 ++++++------------ 1 file changed, 35 insertions(+), 68 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index a83c93fd1c0..5e5f7a617fc 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,83 +2,50 @@ Tests for StatisticsExtractor and related functions """ -import numpy as np -import pytest +import astropy.units as u from astropy.table import QTable +import numpy as np from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor -@pytest.fixture(name="test_plainextractor") -def fixture_test_plainextractor(example_subarray): - """test the PlainExtractor""" - return PlainExtractor(subarray=example_subarray, chunk_size=2500) - - -@pytest.fixture(name="test_sigmaclippingextractor") -def fixture_test_sigmaclippingextractor(example_subarray): - """test the SigmaClippingExtractor""" - return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) - - -def test_extractors(test_plainextractor, test_sigmaclippingextractor): - """test basic functionality of the StatisticsExtractors""" - +def test_extractors(example_subarray): + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - - plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) - - -def test_check_outliers(test_sigmaclippingextractor): - """test detection ability of outliers""" - + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False + +def test_check_outliers(example_subarray): + sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers - flatfield_dl1_data[:, 0, 120] = 120.0 - flatfield_dl1_data[:, 1, 67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] - assert sigmaclipping_stats_list[0].median_outliers[1][67] - assert sigmaclipping_stats_list[1].median_outliers[0][120] - assert sigmaclipping_stats_list[1].median_outliers[1][67] - - -def test_check_chunk_shift(test_sigmaclippingextractor): - """test the chunk shift option and the boundary case for the last chunk""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, chunk_shift=2000 - ) - - # check if three chunks are used for the extraction - assert len(sigmaclipping_stats_list) == 3 + flatfield_dl1_data[:,0,120] = 120.0 + flatfield_dl1_data[:,1,67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + + #check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] == True + assert sigmaclipping_stats_list[0].median_outliers[1][67] == True + assert sigmaclipping_stats_list[1].median_outliers[0][120] == True + assert sigmaclipping_stats_list[1].median_outliers[1][67] == True From d0f17ee4252c1b896a4bd15b708a5bd5961052c7 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 3 May 2024 13:54:45 +0200 Subject: [PATCH 149/221] fix lint --- src/ctapipe/calib/camera/extractor.py | 9 --------- src/ctapipe/calib/camera/tests/test_extractors.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 45bbb27c3e1..c64414c8377 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -123,9 +123,6 @@ def _plain_extraction( # std over the sample per pixel pixel_std = np.ma.std(masked_images, axis=0) - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - # outliers from median image_median_outliers = np.logical_or( pixel_median < self.image_median_cut_outliers[0], @@ -206,9 +203,6 @@ def _sigmaclipping_extraction( # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - # median of the event images - image_median = np.ma.median(masked_images, axis=-1) - # mean, median, and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( @@ -226,9 +220,6 @@ def _sigmaclipping_extraction( unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - # only warn for values discard in the sigma clipping, not those from before - outliers = unused_values & (~masked_images.mask) - # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 5e5f7a617fc..19b04c017fe 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,7 +22,7 @@ def test_extractors(example_subarray): plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - assert plain_stats_list[0].mean - 2.0) > 1.5) == False + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False From 077864886d56116a82dee7103e0c7ced1f73098f Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 23 May 2024 11:37:46 +0200 Subject: [PATCH 150/221] I altered the class variables to th evariance statistics extractor --- src/ctapipe/calib/camera/extractor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index c64414c8377..895947f8543 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -145,20 +145,23 @@ class StarVarianceExtractor(StatisticsExtractor): using the startracker functions """ - min_star_magnitude = Float( - 0.1, - help="Minimum magnitude of stars to be used. Set to appropriate value to avoid ", + sigma_clipping_max_sigma = Int( + default_value=4, + help="Maximal value for the sigma clipping outlier removal", + ).tag(config=True) + sigma_clipping_iterations = Int( + default_value=5, + help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) def __init__(): def __call__( - self, variance_table, trigger_table, initial_pointing, PSF_model + self, variance_table ): - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images From b04e6ee650c3d368c9084ce9f42cae7a2a232f4e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 24 May 2024 13:38:36 +0200 Subject: [PATCH 151/221] added a container for mean variance images and fixed docustring --- src/ctapipe/calib/camera/extractor.py | 43 +++++++++++++++++++++++++-- src/ctapipe/containers.py | 9 +++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 895947f8543..88fda8ef7a0 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -141,8 +141,9 @@ def _plain_extraction( class StarVarianceExtractor(StatisticsExtractor): """ - Extracts pointing information from a series of variance images - using the startracker functions + Generating average variance images from a set + of variance images for the startracker + pointing calibration """ sigma_clipping_max_sigma = Int( @@ -160,7 +161,43 @@ def __call__( self, variance_table ): - + image_chunks = ( + variance_table["image"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["image"].data), self.sample_size) + ) + + time_chunks = ( + variance_table["trigger_times"].data[i : i + self.sample_size] + for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) + ) + + stats_list = [] + + for images, times in zip(image_chunks, time_chunks): + + stats_list.append( + self._sigmaclipping_extraction(images, times) + ) + return stats_list + + def _sigmaclipping_extraction( + self, images, times + )->StatisticsContainer: + + pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( + images, + sigma=self.sigma_clipping_max_sigma, + maxiters=self.sigma_clipping_iterations, + cenfunc="mean", + axis=0, + ) + + return StatisticsContainer( + validity_start=times[0], + validity_stop=times[-1], + mean=pixel_mean.filled(np.nan), + median=pixel_median.filled(np.nan) + ) class SigmaClippingExtractor(StatisticsExtractor): """ diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 9a4678401d8..b364aefa0cc 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,6 +57,7 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", + "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -432,6 +433,13 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") +class VarianceStatisticsContainer(Container): + """Store descriptive statistics of a sequence of variance images""" + + validity_start = Field(np.float32(nan), "start of the validity range") + validity_stop = Field(np.float32(nan), "stop of the validity range") + mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") + class ImageStatisticsContainer(Container): """Store descriptive image statistics""" @@ -442,7 +450,6 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") - class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From e7ec4b88d8c0ecea1ad7ab4367b10fd679b66ec1 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 27 May 2024 15:20:53 +0200 Subject: [PATCH 152/221] I changed the container type for the StarVarianceExtractor --- src/ctapipe/calib/camera/extractor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 88fda8ef7a0..a08ae8d0970 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -182,7 +182,7 @@ def __call__( def _sigmaclipping_extraction( self, images, times - )->StatisticsContainer: + )->VarianceStatisticsContainer: pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( images, @@ -192,11 +192,10 @@ def _sigmaclipping_extraction( axis=0, ) - return StatisticsContainer( + return VarianceStatisticsContainer( validity_start=times[0], validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan) + mean=pixel_mean.filled(np.nan) ) class SigmaClippingExtractor(StatisticsExtractor): From 0921641632fc7085dd7f09da9a7116288b705c9d Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:42:35 +0200 Subject: [PATCH 153/221] fix pylint Remove StarVarianceExtractor since is functionality is featured in the existing Extractors --- src/ctapipe/calib/camera/extractor.py | 89 ++++--------------- .../calib/camera/tests/test_extractors.py | 64 +++++++------ src/ctapipe/containers.py | 7 -- 3 files changed, 53 insertions(+), 107 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index a08ae8d0970..f06138cdd14 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -5,15 +5,12 @@ __all__ = [ "StatisticsExtractor", "PlainExtractor", - "StarVarianceExtractor", "SigmaClippingExtractor", ] - from abc import abstractmethod import numpy as np -import scipy.stats from astropy.stats import sigma_clipped_stats from ctapipe.core import TelescopeComponent @@ -22,10 +19,9 @@ Int, List, ) -from ctapipe.coordinates import EngineeringCameraFrame - class StatisticsExtractor(TelescopeComponent): + """Base StatisticsExtractor component""" sample_size = Int( 2500, @@ -33,11 +29,13 @@ class StatisticsExtractor(TelescopeComponent): ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", + help="""Interval of accepted image values \\ + (fraction with respect to camera median value)""", ).tag(config=True) image_std_cut_outliers = List( [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", + help="""Interval (number of std) of accepted image standard deviation \\ + around camera median value""", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): @@ -76,7 +74,6 @@ def __call__( List of extracted statistics and validity ranges """ - pass class PlainExtractor(StatisticsExtractor): @@ -138,66 +135,6 @@ def _plain_extraction( std=pixel_std.filled(np.nan), ) - -class StarVarianceExtractor(StatisticsExtractor): - """ - Generating average variance images from a set - of variance images for the startracker - pointing calibration - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __init__(): - - def __call__( - self, variance_table - ): - - image_chunks = ( - variance_table["image"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["image"].data), self.sample_size) - ) - - time_chunks = ( - variance_table["trigger_times"].data[i : i + self.sample_size] - for i in range(0, len(variance_table["trigger_times"].data), self.sample_size) - ) - - stats_list = [] - - for images, times in zip(image_chunks, time_chunks): - - stats_list.append( - self._sigmaclipping_extraction(images, times) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times - )->VarianceStatisticsContainer: - - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - images, - sigma=self.sigma_clipping_max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - return VarianceStatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan) - ) - class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a sequence of images @@ -275,18 +212,26 @@ def _sigmaclipping_extraction( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[ + :, np.newaxis + ], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + * std_of_pixel_std[:, np.newaxis], ) return StatisticsContainer( diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 19b04c017fe..89c375387e3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,7 +2,6 @@ Tests for StatisticsExtractor and related functions """ -import astropy.units as u from astropy.table import QTable import numpy as np @@ -10,42 +9,51 @@ def test_extractors(example_subarray): + """test basic functionality of the StatisticsExtractors""" + plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=('time', 'image')) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) - + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) == False - - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) == False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) == False - + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + + assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False + assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + + def test_check_outliers(example_subarray): - sigmaclipping_extractor = SigmaClippingExtractor(subarray=example_subarray, sample_size=2500) + """test detection ability of outliers""" + + sigmaclipping_extractor = SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers - flatfield_dl1_data[:,0,120] = 120.0 - flatfield_dl1_data[:,1,67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=('time', 'image')) + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) - - #check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] == True - assert sigmaclipping_stats_list[0].median_outliers[1][67] == True - assert sigmaclipping_stats_list[1].median_outliers[0][120] == True - assert sigmaclipping_stats_list[1].median_outliers[1][67] == True + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] is True + assert sigmaclipping_stats_list[0].median_outliers[1][67] is True + assert sigmaclipping_stats_list[1].median_outliers[0][120] is True + assert sigmaclipping_stats_list[1].median_outliers[1][67] is True diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index b364aefa0cc..bff71facca4 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -57,7 +57,6 @@ "TriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", - "VarianceStatisticsContainer", "ImageStatisticsContainer", "IntensityStatisticsContainer", "PeakTimeStatisticsContainer", @@ -433,12 +432,6 @@ class StatisticsContainer(Container): std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") -class VarianceStatisticsContainer(Container): - """Store descriptive statistics of a sequence of variance images""" - - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") class ImageStatisticsContainer(Container): """Store descriptive image statistics""" From fa65404a16f1ae8bfbfe12eedd98d15625343951 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:45:29 +0200 Subject: [PATCH 154/221] change __call__() to _extract() --- src/ctapipe/calib/camera/extractor.py | 6 +++--- src/ctapipe/calib/camera/tests/test_extractors.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index f06138cdd14..88bd37b766c 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -51,7 +51,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -82,7 +82,7 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: @@ -150,7 +150,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__( + def _extract( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 89c375387e3..d5f082762ed 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -22,8 +22,8 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -50,7 +50,7 @@ def test_check_outliers(example_subarray): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From 08d43dd295ca66c36ddc1b5ca4bcae558c363e00 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 17:59:36 +0200 Subject: [PATCH 155/221] minor renaming --- src/ctapipe/calib/camera/extractor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 88bd37b766c..8afb50a36a9 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -61,7 +61,7 @@ def _extract( Parameters ---------- dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape + dl1 table with images and timestamps stored in a numpy array of shape (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing @@ -141,11 +141,11 @@ class SigmaClippingExtractor(StatisticsExtractor): using astropy's sigma clipping functions """ - sigma_clipping_max_sigma = Int( + max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", ).tag(config=True) - sigma_clipping_iterations = Int( + iterations = Int( default_value=5, help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) @@ -180,11 +180,10 @@ def _sigmaclipping_extraction( masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, + sigma=self.max_sigma, + maxiters=self.iterations, cenfunc="mean", axis=0, ) @@ -194,7 +193,7 @@ def _sigmaclipping_extraction( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 734d03fb5f8a48c67519b6d1ee2b31c34e6602d1 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:15:53 +0200 Subject: [PATCH 156/221] use pytest.fixture for Extractors --- .../calib/camera/tests/test_extractors.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d5f082762ed..d363ee24ff0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -7,14 +7,21 @@ from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor +@pytest.fixture + def test_plainextractor(example_subarray): + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -def test_extractors(example_subarray): +@pytest.fixture + def test_sigmaclippingextractor(example_subarray): + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" - plain_extractor = PlainExtractor(subarray=example_subarray, sample_size=2500) - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) @@ -22,8 +29,10 @@ def test_extractors(example_subarray): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = plain_extractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + dl1_table=flatfield_dl1_table + ) assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False @@ -38,19 +47,18 @@ def test_extractors(example_subarray): assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False -def test_check_outliers(example_subarray): +def test_check_outliers(test_sigmaclippingextractor): """test detection ability of outliers""" - sigmaclipping_extractor = SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) times = np.linspace(60117.911, 60117.9258, num=5000) flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) # insert outliers flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract(dl1_table=flatfield_dl1_table) + sigmaclipping_stats_list = sigmaclipping_extractor._extract( + dl1_table=flatfield_dl1_table + ) # check if outliers where detected correctly assert sigmaclipping_stats_list[0].median_outliers[0][120] is True From ce9e7841896c713d4fc021de491e29908201ea16 Mon Sep 17 00:00:00 2001 From: Tjark Miener Date: Fri, 31 May 2024 18:59:57 +0200 Subject: [PATCH 157/221] reduce duplicated code of the call function --- src/ctapipe/calib/camera/extractor.py | 52 ++++++------------- .../calib/camera/tests/test_extractors.py | 30 ++++++----- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 8afb50a36a9..f52fb4f5751 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -50,8 +50,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - @abstractmethod - def _extract( + def __call__( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: """ @@ -75,17 +74,6 @@ def _extract( List of extracted statistics and validity ranges """ - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] @@ -100,11 +88,23 @@ def _extract( stats_list = [] for images, times in zip(image_chunks, time_chunks): stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) + self._extract(images, times, masked_pixels_of_sample) ) return stats_list - def _plain_extraction( + @abstractmethod + def _extract( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: + pass + +class PlainExtractor(StatisticsExtractor): + """ + Extractor the statistics from a sequence of images + using numpy and scipy functions + """ + + def _extract( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: @@ -151,28 +151,6 @@ class SigmaClippingExtractor(StatisticsExtractor): ).tag(config=True) def _extract( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( self, images, times, masked_pixels_of_sample ) -> StatisticsContainer: diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index d363ee24ff0..06107a6e7b7 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -4,20 +4,22 @@ from astropy.table import QTable import numpy as np - +import pytest from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor -@pytest.fixture - def test_plainextractor(example_subarray): - return PlainExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor( + subarray=example_subarray, sample_size=2500 + ) -@pytest.fixture - def test_sigmaclippingextractor(example_subarray): - return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 - ) +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor( + subarray=example_subarray, sample_size=2500 + ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -29,8 +31,8 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - plain_stats_list = test_plainextractor._extract(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor._extract( + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) @@ -56,7 +58,7 @@ def test_check_outliers(test_sigmaclippingextractor): flatfield_dl1_data[:, 0, 120] = 120.0 flatfield_dl1_data[:, 1, 67] = 120.0 flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = sigmaclipping_extractor._extract( + sigmaclipping_stats_list = test_sigmaclippingextractor( dl1_table=flatfield_dl1_table ) From 233ffb7544e913125892d2a4bc28ca27aba5429c Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 158/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 256 ++++++------------------- src/ctapipe/image/psf_model.py | 25 +-- 2 files changed, 68 insertions(+), 213 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ae0dfdecbbe..085b96ae846 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -13,6 +13,7 @@ import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -23,23 +24,16 @@ ComponentName, Dict, Float, - Int, Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, TableLoader +from ctapipe.io import EventSource -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "PointingCalculator", - "CameraCalibrator", -] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -83,14 +77,13 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - output_path = Path(help="output filename").tag(config=True) + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? def __init__( self, subarray, config=None, parent=None, - stats_extractor=None, **kwargs, ): """ @@ -107,157 +100,89 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) @abstractmethod - def __call__(self, input_url, tel_id): + def __call__(self, data_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated tel_id : int - The telescope id + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties """ + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) -class TwoPassStatisticsCalculator(CalibrationCalculator): + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): """ - Component to calculate statistics from calibration events. + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( + + def __init__( self, - input_url, - tel_id, - col_name="image", + subarray, + config=None, + parent=None, + **kwargs, ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) + def __call__(self, data_url, tel_id): + pass - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously caluclated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ - def _get_slice_range( + def __init__( self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, + subarray, + config=None, + parent=None, + **kwargs, ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - return slice_start, slice_stop + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) class PointingCalculator(CalibrationCalculator): @@ -288,7 +213,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - psf_model_type = TelescopeParameter( + PSFModel_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -303,11 +228,8 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # TODO: Currently not in the dependency list of ctapipe - from astroquery.vizier import Vizier - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self + self.PSFModel_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -341,66 +263,14 @@ def __call__(self, url, tel_id): location=self.location, ) - with TableLoader(url) as loader: - loader.read_telescope_events_by_id( - telescopes=[tel_id], dl1_parameters=True, observation_info=True - ) - stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, gain): - # So, here i need to match up the validity periods of the relative gain to the variance images - gain_to_variance = np.zeros( - len(var_images) - ) # this array will map the gain values to accumulated variance images - - for i in np.arange( - 1, len(var_images) - ): # the first pairing is 0 -> 0, so start at 1 - for j in np.arange(len(gain), 0): - if var_images[i].validity_start > gain[j].validity_start or j == len( - var_images - ): - gain_to_variance[i] = j - break - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square( - gain[gain_to_variance[i]] - ), # Here i will need to adjust the code based on how the containers for gain will work - ) - - return var_images + def _calibrate_varimages(self, varimages, gain): + pass class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 458070b8145..7af526e6c21 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,30 +10,15 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List +from ctapipe.core import TelescopeComponent -class PSFModel: - def __init__(self, **kwargs): - """ - Base component to describe image distortion due to the optics of the different cameras. - """ - @classmethod - def from_name(cls, name, **kwargs): +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): """ - Obtain an instance of a subclass via its name - - Parameters - ---------- - name : str - Name of the subclass to obtain - - Returns - ------- - Instance - Instance of subclass to this class + Base Component to describe image distortion due to the optics of the different cameras. """ - requested_subclass = cls.non_abstract_subclasses()[name] - return requested_subclass(**kwargs) + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod def pdf(self, *args): From 74807c62a4b5f0def30341083badddd2ae61c61b Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 159/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 085b96ae846..43592933d61 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -271,6 +271,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From 36da84209b157ecbad80464c7576a8989236be14 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 160/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 43592933d61..6fe12a774ed 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -116,21 +116,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -213,7 +225,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -229,7 +241,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -269,7 +281,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From 511ebdf9952dffcd493da990255e9f17eca844fc Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 161/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6fe12a774ed..ded47a46483 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -281,9 +281,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From ed13253e0906623de042e6a4ecf7c0ca46b3e97d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 12:02:41 +0200 Subject: [PATCH 162/221] edit description of StatisticsContainer --- src/ctapipe/containers.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index bff71facca4..40bd34b088d 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -424,13 +424,33 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a sequence of images""" - validity_start = Field(np.float32(nan), "start of the validity range") - validity_stop = Field(np.float32(nan), "stop of the validity range") - mean = Field(np.float32(nan), "Channel-wise and pixel-wise mean") - median = Field(np.float32(nan), "Channel-wise and pixel-wise median") - median_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise median outliers") - std = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation") - std_outliers = Field(np.float32(nan), "Channel-wise and pixel-wise standard deviation outliers") + extraction_start = Field(np.float32(nan), "start of the extraction sequence") + extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + mean = Field( + None, + "mean of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median = Field( + None, + "median of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + median_outliers = Field( + None, + "outliers from the median distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) + std = Field( + None, + "standard deviation of a pixel-wise quantity for each channel" + "Type: float; Shape: (n_channels, n_pixel)", + ) + std_outliers = Field( + None, + "outliers from the standard deviation distribution of a pixel-wise quantity for each channel" + "Type: binary mask; Shape: (n_channels, n_pixel)", + ) class ImageStatisticsContainer(Container): @@ -443,6 +463,7 @@ class ImageStatisticsContainer(Container): skewness = Field(nan, "skewness of intensity") kurtosis = Field(nan, "kurtosis of intensity") + class IntensityStatisticsContainer(ImageStatisticsContainer): default_prefix = "intensity" From 732c5f9754b75f097289e854990ffca7feecbd99 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 13:12:54 +0200 Subject: [PATCH 163/221] added feature to shift the extraction sequence allow overlapping extraction sequences --- src/ctapipe/calib/camera/extractor.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index f52fb4f5751..227488813f3 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -51,7 +51,11 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" + self, + dl1_table, + masked_pixels_of_sample=None, + sample_shift=None, + col_name="image", ) -> list: """ Call the relevant functions to extract the statistics @@ -64,6 +68,8 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing + sample_shift : int + number of samples to shift the extraction sequence col_name : string column name in the dl1 table @@ -74,14 +80,19 @@ def __call__( List of extracted statistics and validity ranges """ + # If no sample_shift is provided, the sample_shift is set to self.sample_size + # meaning that the samples are not overlapping. + if sample_shift is None: + sample_shift = self.sample_size + # in python 3.12 itertools.batched can be used image_chunks = ( dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) + for i in range(0, len(dl1_table[col_name].data), sample_shift) ) time_chunks = ( dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) + for i in range(0, len(dl1_table["time"]), sample_shift) ) # Calculate the statistics from a sequence of images @@ -171,7 +182,9 @@ def _extract( pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > (self.max_sigma * pixel_std) + unused_values = np.abs(masked_images - pixel_mean) > ( + self.max_sigma * pixel_std + ) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values From 65ded4068e703734460247d3d51b13344790b4a0 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 14:23:53 +0200 Subject: [PATCH 164/221] fix boundary case for the last chunk renaming to chunk(s) and chunk_size and _shift added test for chunk_shift and boundary case --- src/ctapipe/calib/camera/extractor.py | 74 ++++++++++--------- .../calib/camera/tests/test_extractors.py | 21 +++++- src/ctapipe/containers.py | 6 +- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 227488813f3..86d9c1345fd 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,5 +1,5 @@ """ -Extraction algorithms to compute the statistics from a sequence of images +Extraction algorithms to compute the statistics from a chunk of images """ __all__ = [ @@ -23,9 +23,9 @@ class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" - sample_size = Int( + chunk_size = Int( 2500, - help="Size of the sample used for the calculation of the statistical values", + help="Size of the chunk used for the calculation of the statistical values", ).tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], @@ -41,8 +41,7 @@ class StatisticsExtractor(TelescopeComponent): def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). ->>>>>>> 58d868c8 (added stats extractor parent component) + from a chunk of charges and pulse times (images). Parameters ---------- @@ -54,7 +53,7 @@ def __call__( self, dl1_table, masked_pixels_of_sample=None, - sample_shift=None, + chunk_shift=None, col_name="image", ) -> list: """ @@ -68,39 +67,44 @@ def __call__( (n_images, n_channels, n_pix). masked_pixels_of_sample : ndarray boolean array of masked pixels that are not available for processing - sample_shift : int - number of samples to shift the extraction sequence + chunk_shift : int + number of samples to shift the extraction chunk col_name : string column name in the dl1 table Returns ------- List StatisticsContainer: - - List of extracted statistics and validity ranges + List of extracted statistics and extraction chunks """ - # If no sample_shift is provided, the sample_shift is set to self.sample_size - # meaning that the samples are not overlapping. - if sample_shift is None: - sample_shift = self.sample_size - - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), sample_shift) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), sample_shift) - ) - - # Calculate the statistics from a sequence of images + # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size + # meaning that the extraction chunks are not overlapping. + if chunk_shift is None: + chunk_shift = self.chunk_size + + # Function to split table data into appropriated chunks + def _get_chunks(col_name): + return [ + ( + dl1_table[col_name].data[i : i + self.chunk_size] + if i + self.chunk_size <= len(dl1_table[col_name]) + else dl1_table[col_name].data[ + len(dl1_table[col_name].data) + - self.chunk_size : len(dl1_table[col_name].data) + ] + ) + for i in range(0, len(dl1_table[col_name].data), chunk_shift) + ] + + # Get the chunks for the timestamps and selected column name + time_chunks = _get_chunks("time") + image_chunks = _get_chunks(col_name) + + # Calculate the statistics from a chunk of images stats_list = [] for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._extract(images, times, masked_pixels_of_sample) - ) + stats_list.append(self._extract(images, times, masked_pixels_of_sample)) return stats_list @abstractmethod @@ -111,7 +115,7 @@ def _extract( class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a sequence of images + Extractor the statistics from a chunk of images using numpy and scipy functions """ @@ -138,8 +142,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), @@ -148,7 +152,7 @@ def _extract( class SigmaClippingExtractor(StatisticsExtractor): """ - Extracts the statistics from a sequence of images + Extracts the statistics from a chunk of images using astropy's sigma clipping functions """ @@ -225,8 +229,8 @@ def _extract( ) return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], + extraction_start=times[0], + extraction_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 06107a6e7b7..40efd4f2fc3 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -11,14 +11,14 @@ def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" return PlainExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" return SigmaClippingExtractor( - subarray=example_subarray, sample_size=2500 + subarray=example_subarray, chunk_size=2500 ) def test_extractors(test_plainextractor, test_sigmaclippingextractor): @@ -67,3 +67,20 @@ def test_check_outliers(test_sigmaclippingextractor): assert sigmaclipping_stats_list[0].median_outliers[1][67] is True assert sigmaclipping_stats_list[1].median_outliers[0][120] is True assert sigmaclipping_stats_list[1].median_outliers[1][67] is True + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, + chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 + diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index 40bd34b088d..a937f49d337 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -422,10 +422,10 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): - """Store descriptive statistics of a sequence of images""" + """Store descriptive statistics of a chunk of images""" - extraction_start = Field(np.float32(nan), "start of the extraction sequence") - extraction_stop = Field(np.float32(nan), "stop of the extraction sequence") + extraction_start = Field(np.float32(nan), "start of the extraction chunk") + extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") mean = Field( None, "mean of a pixel-wise quantity for each channel" From a505ae633473c0abacba5a9a42ed4f03e3ede182 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:13:27 +0200 Subject: [PATCH 165/221] fix tests --- .../calib/camera/tests/test_extractors.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py index 40efd4f2fc3..a83c93fd1c0 100644 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -2,24 +2,24 @@ Tests for StatisticsExtractor and related functions """ -from astropy.table import QTable import numpy as np import pytest +from astropy.table import QTable + from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + @pytest.fixture(name="test_plainextractor") def fixture_test_plainextractor(example_subarray): """test the PlainExtractor""" - return PlainExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + @pytest.fixture(name="test_sigmaclippingextractor") def fixture_test_sigmaclippingextractor(example_subarray): """test the SigmaClippingExtractor""" - return SigmaClippingExtractor( - subarray=example_subarray, chunk_size=2500 - ) + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + def test_extractors(test_plainextractor, test_sigmaclippingextractor): """test basic functionality of the StatisticsExtractors""" @@ -36,17 +36,17 @@ def test_extractors(test_plainextractor, test_sigmaclippingextractor): dl1_table=flatfield_dl1_table ) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - assert np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) is False - assert np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) is False + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) def test_check_outliers(test_sigmaclippingextractor): @@ -63,11 +63,11 @@ def test_check_outliers(test_sigmaclippingextractor): ) # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] is True - assert sigmaclipping_stats_list[0].median_outliers[1][67] is True - assert sigmaclipping_stats_list[1].median_outliers[0][120] is True - assert sigmaclipping_stats_list[1].median_outliers[1][67] is True - + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + def test_check_chunk_shift(test_sigmaclippingextractor): """test the chunk shift option and the boundary case for the last chunk""" @@ -77,10 +77,8 @@ def test_check_chunk_shift(test_sigmaclippingextractor): # insert outliers flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, - chunk_shift=2000 + dl1_table=flatfield_dl1_table, chunk_shift=2000 ) # check if three chunks are used for the extraction assert len(sigmaclipping_stats_list) == 3 - From fa6c65adc06b9b0ebc099b874aea339c0d0a2304 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 12 Jun 2024 15:26:03 +0200 Subject: [PATCH 166/221] fix ruff --- src/ctapipe/calib/camera/extractor.py | 40 +++++++++------------- src/ctapipe/image/tests/test_statistics.py | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 86d9c1345fd..4c8f49d1f38 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -13,13 +13,14 @@ import numpy as np from astropy.stats import sigma_clipped_stats -from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Int, List, ) + class StatisticsExtractor(TelescopeComponent): """Base StatisticsExtractor component""" @@ -90,8 +91,9 @@ def _get_chunks(col_name): dl1_table[col_name].data[i : i + self.chunk_size] if i + self.chunk_size <= len(dl1_table[col_name]) else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - - self.chunk_size : len(dl1_table[col_name].data) + len(dl1_table[col_name].data) - self.chunk_size : len( + dl1_table[col_name].data + ) ] ) for i in range(0, len(dl1_table[col_name].data), chunk_shift) @@ -108,21 +110,17 @@ def _get_chunks(col_name): return stats_list @abstractmethod - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: pass + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a chunk of images using numpy and scipy functions """ - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -150,6 +148,7 @@ def _extract( std=pixel_std.filled(np.nan), ) + class SigmaClippingExtractor(StatisticsExtractor): """ Extracts the statistics from a chunk of images @@ -165,10 +164,7 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def _extract( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - + def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: # ensure numpy array masked_images = np.ma.array(images, mask=masked_pixels_of_sample) @@ -206,25 +202,21 @@ def _extract( image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] image_median_outliers = np.logical_or( image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[ - :, np.newaxis - ], + > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object + * median_of_pixel_median[:, np.newaxis], ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] image_std_outliers = np.logical_or( deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object + < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object + > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object * std_of_pixel_std[:, np.newaxis], ) diff --git a/src/ctapipe/image/tests/test_statistics.py b/src/ctapipe/image/tests/test_statistics.py index 23806705787..4403e05ca0a 100644 --- a/src/ctapipe/image/tests/test_statistics.py +++ b/src/ctapipe/image/tests/test_statistics.py @@ -49,7 +49,7 @@ def test_kurtosis(): def test_return_type(): - from ctapipe.containers import PeakTimeStatisticsContainer, ImageStatisticsContainer + from ctapipe.containers import ImageStatisticsContainer, PeakTimeStatisticsContainer from ctapipe.image import descriptive_statistics rng = np.random.default_rng(0) From e1d5d6d963198edd68f282d9b11928e2b159c240 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 167/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ded47a46483..518da78d37a 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -31,7 +31,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -275,6 +275,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -299,7 +304,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From f9f0d03f578f0f7d7ae8e421eb47af0c7eaae415 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 168/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 222 +++++++++++++++++-------- 1 file changed, 155 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 518da78d37a..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -13,7 +13,6 @@ import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -25,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -33,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -77,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -100,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. - """ - - def _check_req_data(self, url, tel_id, calibration_type): + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -240,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -286,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From 1616d4d97cc6517f8e1473038716c8c2a13e9e57 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 169/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From 4cb94034d64ed11084d44c26d26bafab9d08a7e5 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 7 Aug 2024 21:18:05 +0200 Subject: [PATCH 170/221] solved merge conflicts --- src/ctapipe/calib/camera/extractor.py | 171 ++++++++++++-------------- 1 file changed, 79 insertions(+), 92 deletions(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 4c8f49d1f38..7699f8cc83d 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,5 +1,9 @@ """ +<<<<<<< HEAD Extraction algorithms to compute the statistics from a chunk of images +======= +Extraction algorithms to compute the statistics from a sequence of images +>>>>>>> f7d3223e (solved merge conflicts) """ __all__ = [ @@ -8,13 +12,15 @@ "SigmaClippingExtractor", ] + from abc import abstractmethod import numpy as np +import scipy.stats from astropy.stats import sigma_clipped_stats -from ctapipe.containers import StatisticsContainer from ctapipe.core import TelescopeComponent +from ctapipe.containers import StatisticsContainer from ctapipe.core.traits import ( Int, List, @@ -22,27 +28,21 @@ class StatisticsExtractor(TelescopeComponent): - """Base StatisticsExtractor component""" - chunk_size = Int( - 2500, - help="Size of the chunk used for the calculation of the statistical values", - ).tag(config=True) + sample_size = Int(2500, help="sample size").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], - help="""Interval of accepted image values \\ - (fraction with respect to camera median value)""", + help="Interval of accepted image values (fraction with respect to camera median value)", ).tag(config=True) image_std_cut_outliers = List( [-3, 3], - help="""Interval (number of std) of accepted image standard deviation \\ - around camera median value""", + help="Interval (number of std) of accepted image standard deviation around camera median value", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): """ Base component to handle the extraction of the statistics - from a chunk of charges and pulse times (images). + from a sequence of charges and pulse times (images). Parameters ---------- @@ -50,13 +50,8 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__( - self, - dl1_table, - masked_pixels_of_sample=None, - chunk_shift=None, - col_name="image", - ) -> list: + @abstractmethod + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -64,65 +59,42 @@ def __call__( Parameters ---------- dl1_table : ndarray - dl1 table with images and timestamps stored in a numpy array of shape + dl1 table with images and times stored in a numpy array of shape (n_images, n_channels, n_pix). - masked_pixels_of_sample : ndarray - boolean array of masked pixels that are not available for processing - chunk_shift : int - number of samples to shift the extraction chunk col_name : string column name in the dl1 table Returns ------- List StatisticsContainer: - List of extracted statistics and extraction chunks + List of extracted statistics and validity ranges """ - # If no chunk_shift is provided, the chunk_shift is set to self.chunk_size - # meaning that the extraction chunks are not overlapping. - if chunk_shift is None: - chunk_shift = self.chunk_size - - # Function to split table data into appropriated chunks - def _get_chunks(col_name): - return [ - ( - dl1_table[col_name].data[i : i + self.chunk_size] - if i + self.chunk_size <= len(dl1_table[col_name]) - else dl1_table[col_name].data[ - len(dl1_table[col_name].data) - self.chunk_size : len( - dl1_table[col_name].data - ) - ] - ) - for i in range(0, len(dl1_table[col_name].data), chunk_shift) - ] - - # Get the chunks for the timestamps and selected column name - time_chunks = _get_chunks("time") - image_chunks = _get_chunks(col_name) - - # Calculate the statistics from a chunk of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append(self._extract(images, times, masked_pixels_of_sample)) - return stats_list - - @abstractmethod - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - pass - - class PlainExtractor(StatisticsExtractor): """ - Extractor the statistics from a chunk of images + Extractor the statistics from a sequence of images using numpy and scipy functions """ - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + return stats_list + + def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -133,6 +105,9 @@ def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContaine # std over the sample per pixel pixel_std = np.ma.std(masked_images, axis=0) + # median of the median over the camera + median_of_pixel_median = np.ma.median(pixel_median, axis=1) + # outliers from median image_median_outliers = np.logical_or( pixel_median < self.image_median_cut_outliers[0], @@ -140,8 +115,8 @@ def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContaine ) return StatisticsContainer( - extraction_start=times[0], - extraction_stop=times[-1], + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), @@ -151,28 +126,48 @@ def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContaine class SigmaClippingExtractor(StatisticsExtractor): """ - Extracts the statistics from a chunk of images + Extractor the statistics from a sequence of images using astropy's sigma clipping functions """ - max_sigma = Int( + sigma_clipping_max_sigma = Int( default_value=4, help="Maximal value for the sigma clipping outlier removal", ).tag(config=True) - iterations = Int( + sigma_clipping_iterations = Int( default_value=5, help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + + # in python 3.12 itertools.batched can be used + image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) + time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + + # Calculate the statistics from a sequence of images + stats_list = [] + for images, times in zip(image_chunks,time_chunks): + stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + return stats_list + + def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: + # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) + masked_images = np.ma.array( + images, + mask=masked_pixels_of_sample + ) + + # median of the event images + image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel + max_sigma = self.sigma_clipping_max_sigma pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( masked_images, - sigma=self.max_sigma, - maxiters=self.iterations, + sigma=max_sigma, + maxiters=self.sigma_clipping_iterations, cenfunc="mean", axis=0, ) @@ -182,9 +177,10 @@ def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContaine pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - unused_values = np.abs(masked_images - pixel_mean) > ( - self.max_sigma * pixel_std - ) + unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) + + # only warn for values discard in the sigma clipping, not those from before + outliers = unused_values & (~masked_images.mask) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values @@ -200,29 +196,20 @@ def _extract(self, images, times, masked_pixels_of_sample) -> StatisticsContaine # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = np.logical_or( - image_deviation - < self.image_median_cut_outliers[0] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[:, np.newaxis], - image_deviation - > self.image_median_cut_outliers[1] # pylint: disable=unsubscriptable-object - * median_of_pixel_median[:, np.newaxis], - ) + + image_median_outliers = ( + np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], + image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = np.logical_or( - deviation - < self.image_std_cut_outliers[0] # pylint: disable=unsubscriptable-object - * std_of_pixel_std[:, np.newaxis], - deviation - > self.image_std_cut_outliers[1] # pylint: disable=unsubscriptable-object - * std_of_pixel_std[:, np.newaxis], - ) + image_std_outliers = ( + np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) return StatisticsContainer( - extraction_start=times[0], - extraction_stop=times[-1], + validity_start=times[0], + validity_stop=times[-1], mean=pixel_mean.filled(np.nan), median=pixel_median.filled(np.nan), median_outliers=image_median_outliers.filled(True), From 03b263735e431ce9b9ca22a6c01d08f32164c3a2 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 14:29:55 +0200 Subject: [PATCH 171/221] I made prototypes for the CalibrationCalculators --- src/ctapipe/calib/camera/calibrator.py | 256 ++++++------------------- src/ctapipe/image/psf_model.py | 25 +-- 2 files changed, 68 insertions(+), 213 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ae0dfdecbbe..085b96ae846 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -13,6 +13,7 @@ import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -23,23 +24,16 @@ ComponentName, Dict, Float, - Int, Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, TableLoader +from ctapipe.io import EventSource -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "PointingCalculator", - "CameraCalibrator", -] +__all__ = ["CameraCalibrator", "CalibrationCalculator"] @cache @@ -83,14 +77,13 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - output_path = Path(help="output filename").tag(config=True) + # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? def __init__( self, subarray, config=None, parent=None, - stats_extractor=None, **kwargs, ): """ @@ -107,157 +100,89 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor + self.stats_extractor = StatisticsExtractor.from_name( + self.stats_extractor_type, subarray=self.subarray, parent=self + ) @abstractmethod - def __call__(self, input_url, tel_id): + def __call__(self, data_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated + Source : EventSource + EventSource containing the events interleaved calibration events + from which the coefficients are to be calculated tel_id : int - The telescope id + The telescope id. Used to obtain to correct traitlet configuration + and instrument properties """ + def _check_req_data(self, url, tel_id, caltype): + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + caldata = getattr(event.mon.tel[tel_id], caltype) -class TwoPassStatisticsCalculator(CalibrationCalculator): + if caldata is None: + return False + + return True + + +class PedestalCalculator(CalibrationCalculator): """ - Component to calculate statistics from calibration events. + Component to calculate pedestals from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( + + def __init__( self, - input_url, - tel_id, - col_name="image", + subarray, + config=None, + parent=None, + **kwargs, ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) + def __call__(self, data_url, tel_id): + pass - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously caluclated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) +class GainCalculator(CalibrationCalculator): + """ + Component to calculate the relative gain from interleaved flatfield events. + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ - def _get_slice_range( + def __init__( self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, + subarray, + config=None, + parent=None, + **kwargs, ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - return slice_start, slice_stop + def __call__(self, data_url, tel_id): + if self._check_req_data(data_url, tel_id, "pedestal"): + raise KeyError( + "Pedestals not found. Pedestal calculation needs to be performed first." + ) class PointingCalculator(CalibrationCalculator): @@ -288,7 +213,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - psf_model_type = TelescopeParameter( + PSFModel_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -303,11 +228,8 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # TODO: Currently not in the dependency list of ctapipe - from astroquery.vizier import Vizier - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self + self.PSFModel_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -341,66 +263,14 @@ def __call__(self, url, tel_id): location=self.location, ) - with TableLoader(url) as loader: - loader.read_telescope_events_by_id( - telescopes=[tel_id], dl1_parameters=True, observation_info=True - ) - stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, gain): - # So, here i need to match up the validity periods of the relative gain to the variance images - gain_to_variance = np.zeros( - len(var_images) - ) # this array will map the gain values to accumulated variance images - - for i in np.arange( - 1, len(var_images) - ): # the first pairing is 0 -> 0, so start at 1 - for j in np.arange(len(gain), 0): - if var_images[i].validity_start > gain[j].validity_start or j == len( - var_images - ): - gain_to_variance[i] = j - break - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square( - gain[gain_to_variance[i]] - ), # Here i will need to adjust the code based on how the containers for gain will work - ) - - return var_images + def _calibrate_varimages(self, varimages, gain): + pass class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 458070b8145..7af526e6c21 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,30 +10,15 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List +from ctapipe.core import TelescopeComponent -class PSFModel: - def __init__(self, **kwargs): - """ - Base component to describe image distortion due to the optics of the different cameras. - """ - @classmethod - def from_name(cls, name, **kwargs): +class PSFModel(TelescopeComponent): + def __init__(self, subarray, config=None, parent=None, **kwargs): """ - Obtain an instance of a subclass via its name - - Parameters - ---------- - name : str - Name of the subclass to obtain - - Returns - ------- - Instance - Instance of subclass to this class + Base Component to describe image distortion due to the optics of the different cameras. """ - requested_subclass = cls.non_abstract_subclasses()[name] - return requested_subclass(**kwargs) + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod def pdf(self, *args): From 0e85e8237f7ff77c1eed356cb5035ee68ca125dc Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:07:12 +0200 Subject: [PATCH 172/221] I made PSFModel a generic class --- src/ctapipe/calib/camera/calibrator.py | 1 + src/ctapipe/image/psf_model.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 085b96ae846..43592933d61 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -271,6 +271,7 @@ def __call__(self, url, tel_id): def _calibrate_varimages(self, varimages, gain): pass + # So, here i need to match up the validity periods of the relative gain to the variance images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 7af526e6c21..bf962135b97 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -10,15 +10,30 @@ from scipy.stats import laplace, laplace_asymmetric from traitlets import List -from ctapipe.core import TelescopeComponent - -class PSFModel(TelescopeComponent): - def __init__(self, subarray, config=None, parent=None, **kwargs): +class PSFModel: + def __init__(self, **kwargs): """ Base Component to describe image distortion due to the optics of the different cameras. """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) @abstractmethod def pdf(self, *args): From d3b5d0b7490a73a612497417c79a6367f0fc1dea Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 12 Jun 2024 15:58:56 +0200 Subject: [PATCH 173/221] I fixed some variable names --- src/ctapipe/calib/camera/calibrator.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 43592933d61..6fe12a774ed 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -116,21 +116,33 @@ def __call__(self, data_url, tel_id): Parameters ---------- - Source : EventSource - EventSource containing the events interleaved calibration events - from which the coefficients are to be calculated + data_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated tel_id : int - The telescope id. Used to obtain to correct traitlet configuration - and instrument properties + The telescope id. """ - def _check_req_data(self, url, tel_id, caltype): + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ with EventSource(url, max_events=1) as source: event = next(iter(source)) - caldata = getattr(event.mon.tel[tel_id], caltype) + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - if caldata is None: + if calibration_data is None: return False return True @@ -213,7 +225,7 @@ class PointingCalculator(CalibrationCalculator): 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" ).tag(config=True) - PSFModel_type = TelescopeParameter( + psf_model_type = TelescopeParameter( trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="PlainExtractor", help="Name of the PSFModel Subclass to be used.", @@ -229,7 +241,7 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.psf = PSFModel.from_name( - self.PSFModel_type, subarray=self.subarray, parent=self + self.pas_model_type, subarray=self.subarray, parent=self ) self.location = EarthLocation( @@ -269,7 +281,7 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_varimages(self, varimages, gain): + def _calibrate_var_images(self, varimages, gain): pass # So, here i need to match up the validity periods of the relative gain to the variance images From a5b3a020ee97d4a30082bfcfa8685e1074ddfad5 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 13 Jun 2024 09:44:37 +0200 Subject: [PATCH 174/221] Added a method for calibrating variance images --- src/ctapipe/calib/camera/calibrator.py | 23 +++++++++++++++++++++-- src/ctapipe/image/psf_model.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6fe12a774ed..ded47a46483 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -281,9 +281,28 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - def _calibrate_var_images(self, varimages, gain): - pass + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images + gain_to_variance = np.zeros( + len(var_images) + ) # this array will map the gain values to accumulated variance images + + for i in np.arange( + 1, len(var_images) + ): # the first pairing is 0 -> 0, so start at 1 + for j in np.arange(len(gain), 0): + if var_images[i].validity_start > gain[j].validity_start or j == len( + var_images + ): + gain_to_variance[i] = j + break + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, np.square(gain[gain_to_variance[i]]) + ) + + return var_images class CameraCalibrator(TelescopeComponent): diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index bf962135b97..458070b8145 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -14,7 +14,7 @@ class PSFModel: def __init__(self, **kwargs): """ - Base Component to describe image distortion due to the optics of the different cameras. + Base component to describe image distortion due to the optics of the different cameras. """ @classmethod From 4478aedceb1b51730231fab4c666a2504e7bc02c Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 14 Jun 2024 09:41:52 +0200 Subject: [PATCH 175/221] Commit before push for tjark --- src/ctapipe/calib/camera/calibrator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ded47a46483..518da78d37a 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -31,7 +31,7 @@ from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource +from ctapipe.io import EventSource, TableLoader __all__ = ["CameraCalibrator", "CalibrationCalculator"] @@ -275,6 +275,11 @@ def __call__(self, url, tel_id): location=self.location, ) + with TableLoader(url) as loader: + loader.read_telescope_events_by_id( + telescopes=[tel_id], dl1_parameters=True, observation_info=True + ) + stars_in_fov = Vizier.query_region( self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" )[0] @@ -299,7 +304,10 @@ def _calibrate_var_images(self, var_images, gain): for i, var_image in enumerate(var_images): var_images[i].image = np.divide( - var_image.image, np.square(gain[gain_to_variance[i]]) + var_image.image, + np.square( + gain[gain_to_variance[i]] + ), # Here i will need to adjust the code based on how the containers for gain will work ) return var_images From 4d41ad838823c95df3a80b99e2b0c84e16378d8f Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 28 Jun 2024 18:23:18 +0200 Subject: [PATCH 176/221] added StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 222 +++++++++++++++++-------- 1 file changed, 155 insertions(+), 67 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 518da78d37a..6411c43320c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -13,7 +13,6 @@ import numpy as np from astropy.coordinates import Angle, EarthLocation, SkyCoord -from astroquery.vizier import Vizier from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -25,6 +24,7 @@ Dict, Float, Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor @@ -33,7 +33,12 @@ from ctapipe.image.reducer import DataVolumeReducer from ctapipe.io import EventSource, TableLoader -__all__ = ["CameraCalibrator", "CalibrationCalculator"] +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] @cache @@ -77,13 +82,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsExtractor subclass to be used.", ).tag(config=True) - # sample_size, how do i do this without copying the StatisticsExtractor traitlets? is this needed? + output_path = Path(help="output filename").tag(config=True) def __init__( self, subarray, config=None, parent=None, + stats_extractor=None, **kwargs, ): """ @@ -100,101 +106,156 @@ def __init__( parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.stats_extractor = StatisticsExtractor.from_name( - self.stats_extractor_type, subarray=self.subarray, parent=self - ) + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, data_url, tel_id): + def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): """ Call the relevant functions to calculate the calibration coefficients for a given set of events Parameters ---------- - data_url : str + input_url : str URL where the events are stored from which the calibration coefficients are to be calculated tel_id : int - The telescope id. - """ - - def _check_req_data(self, url, tel_id, calibration_type): + The telescope id + faulty_pixels_threshold: float + percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk + chunk_shift : int + number of samples to shift the extraction chunk """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True -class PedestalCalculator(CalibrationCalculator): +class StatisticsCalculator(CalibrationCalculator): """ - Component to calculate pedestals from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + Component to calculate statistics from calibration events. """ - def __init__( + def __call__( self, - subarray, - config=None, - parent=None, - **kwargs, + input_url, + tel_id, + col_name="image", + faulty_pixels_threshold=0.1, + chunk_shift=100, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - def __call__(self, data_url, tel_id): - pass + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) -class GainCalculator(CalibrationCalculator): - """ - Component to calculate the relative gain from interleaved flatfield events. + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) - def __init__( + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + chunk_shift=chunk_shift, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously caluclated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + + def _get_slice_range( self, - subarray, - config=None, - parent=None, - **kwargs, + chunk_nr, + chunk_size, + chunk_shift, + faultless_previous_chunk, + last_chunk, + last_element, ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - def __call__(self, data_url, tel_id): - if self._check_req_data(data_url, tel_id, "pedestal"): - raise KeyError( - "Pedestals not found. Pedestal calculation needs to be performed first." + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + chunk_shift ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + + return slice_start, slice_stop class PointingCalculator(CalibrationCalculator): @@ -240,6 +301,9 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + # TODO: Currently not in the dependency list of ctapipe + from astroquery.vizier import Vizier + self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) @@ -286,6 +350,30 @@ def __call__(self, url, tel_id): stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + def _calibrate_var_images(self, var_images, gain): # So, here i need to match up the validity periods of the relative gain to the variance images gain_to_variance = np.zeros( From 2b9dccdb2bc9dc6504085698272ea0a2278b438b Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 5 Jul 2024 13:56:45 +0200 Subject: [PATCH 177/221] make faulty_pixels_threshold and chunk_shift as traits rename stats calculator to TwoPass... --- src/ctapipe/calib/camera/calibrator.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6411c43320c..ae0dfdecbbe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -23,6 +23,7 @@ ComponentName, Dict, Float, + Int, Integer, Path, TelescopeParameter, @@ -35,7 +36,7 @@ __all__ = [ "CalibrationCalculator", - "StatisticsCalculator", + "TwoPassStatisticsCalculator", "PointingCalculator", "CameraCalibrator", ] @@ -126,7 +127,7 @@ def __init__( self.stats_extractor[name] = stats_extractor @abstractmethod - def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): + def __call__(self, input_url, tel_id): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -138,25 +139,28 @@ def __call__(self, input_url, tel_id, faulty_pixels_threshold, chunk_shift): are to be calculated tel_id : int The telescope id - faulty_pixels_threshold: float - percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk - chunk_shift : int - number of samples to shift the extraction chunk """ -class StatisticsCalculator(CalibrationCalculator): +class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + def __call__( self, input_url, tel_id, col_name="image", - faulty_pixels_threshold=0.1, - chunk_shift=100, ): # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader @@ -200,12 +204,11 @@ def __call__( # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. if ( np.count_nonzero(outlier_mask) / len(outlier_mask) - > faulty_pixels_threshold + > self.faulty_pixels_threshold ): slice_start, slice_stop = self._get_slice_range( chunk_nr=chunk_nr, chunk_size=extractor.chunk_size, - chunk_shift=chunk_shift, faultless_previous_chunk=faultless_previous_chunk, last_chunk=len(stats_list_firstpass) - 1, last_element=len(dl1_table[tel_id]) - 1, @@ -217,7 +220,7 @@ def __call__( # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=chunk_shift + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift ) # Extend the final stats list by the stats list of the second pass. stats_list.extend(stats_list_secondpass) @@ -239,7 +242,6 @@ def _get_slice_range( self, chunk_nr, chunk_size, - chunk_shift, faultless_previous_chunk, last_chunk, last_element, @@ -247,13 +249,13 @@ def _get_slice_range( slice_start = 0 if chunk_nr > 0: slice_start = ( - chunk_size * (chunk_nr - 1) + chunk_shift + chunk_size * (chunk_nr - 1) + self.chunk_shift if faultless_previous_chunk - else chunk_size * chunk_nr + chunk_shift + else chunk_size * chunk_nr + self.chunk_shift ) slice_stop = last_element if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - chunk_shift - 1 + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 return slice_start, slice_stop From de337cba307e5507f9efc35b2e2ca956747165e4 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 12:15:10 +0200 Subject: [PATCH 178/221] Removed Pointing Calculator --- src/ctapipe/calib/camera/calibrator.py | 164 +------------------------ 1 file changed, 5 insertions(+), 159 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index ae0dfdecbbe..07cd295d06c 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -3,16 +3,12 @@ calibration and image extraction, as well as supporting algorithms. """ +import pickle from abc import abstractmethod from functools import cache -import pathlib import astropy.units as u -from astropy.table import Table -import pickle - import numpy as np -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 from ctapipe.calib.camera.extractor import StatisticsExtractor @@ -21,23 +17,19 @@ from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, Float, Int, - Integer, Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, TableLoader +from ctapipe.io import TableLoader __all__ = [ "CalibrationCalculator", "TwoPassStatisticsCalculator", - "PointingCalculator", "CameraCalibrator", ] @@ -146,7 +138,7 @@ class TwoPassStatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. """ - + faulty_pixels_threshold = Float( 0.1, help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", @@ -155,14 +147,13 @@ class TwoPassStatisticsCalculator(CalibrationCalculator): 100, help="Number of samples to shift the extraction chunk for the calculation of the statistical values", ).tag(config=True) - + def __call__( self, input_url, tel_id, col_name="image", ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( @@ -187,7 +178,6 @@ def __call__( stats_list = [] faultless_previous_chunk = False for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk if faultless_previous_chunk: stats_list.append(stats_list_firstpass[chunk_nr - 1]) @@ -215,7 +205,7 @@ def __call__( ) # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously caluclated start and stop. + # Slice the dl1 table according to the previously calculated start and stop. dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] # Run the stats extractor on the sliced dl1 table with a chunk_shift # to remove the period of trouble (carflashes etc.) as effectively as possible. @@ -237,7 +227,6 @@ def __call__( with open(self.output_path, "wb") as f: pickle.dump(stats_list, f) - def _get_slice_range( self, chunk_nr, @@ -260,149 +249,6 @@ def _get_slice_range( return slice_start, slice_stop -class PointingCalculator(CalibrationCalculator): - """ - Component to calculate pointing corrections from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - telescope_location: dict - The location of the telescope for which the pointing correction is to be calculated - """ - - telescope_location = Dict( - {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, - help="Telescope location, longitude and latitude should be expressed in deg, " - "elevation - in meters", - ).tag(config=True) - - min_star_prominence = Integer( - 3, - help="Minimal star prominence over the background in terms of " - "NSB variance std deviations", - ).tag(config=True) - - max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" - ).tag(config=True) - - psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), - default_value="PlainExtractor", - help="Name of the PSFModel Subclass to be used.", - ).tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - **kwargs, - ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - # TODO: Currently not in the dependency list of ctapipe - from astroquery.vizier import Vizier - - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self - ) - - self.location = EarthLocation( - lon=self.telescope_location["longitude"] * u.deg, - lat=self.telescope_location["latitude"] * u.deg, - height=self.telescope_location["elevation"] * u.m, - ) - - def __call__(self, url, tel_id): - if self._check_req_data(url, tel_id, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - self.tel_id = tel_id - - with EventSource(url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) - - with TableLoader(url) as loader: - loader.read_telescope_events_by_id( - telescopes=[tel_id], dl1_parameters=True, observation_info=True - ) - - stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] - - stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, gain): - # So, here i need to match up the validity periods of the relative gain to the variance images - gain_to_variance = np.zeros( - len(var_images) - ) # this array will map the gain values to accumulated variance images - - for i in np.arange( - 1, len(var_images) - ): # the first pairing is 0 -> 0, so start at 1 - for j in np.arange(len(gain), 0): - if var_images[i].validity_start > gain[j].validity_start or j == len( - var_images - ): - gain_to_variance[i] = j - break - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square( - gain[gain_to_variance[i]] - ), # Here i will need to adjust the code based on how the containers for gain will work - ) - - return var_images - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill From b1f591d4b387a678a032911a35fe18e13c6f0d74 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:35:09 +0200 Subject: [PATCH 179/221] Removing PointingCalculator, PSF model and interpolators --- src/ctapipe/image/psf_model.py | 95 ------------------- src/ctapipe/monitoring/interpolation.py | 6 +- .../monitoring/tests/test_interpolator.py | 4 +- 3 files changed, 4 insertions(+), 101 deletions(-) delete mode 100644 src/ctapipe/image/psf_model.py diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py deleted file mode 100644 index 458070b8145..00000000000 --- a/src/ctapipe/image/psf_model.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Models for the Point Spread Functions of the different telescopes -""" - -__all__ = ["PSFModel", "ComaModel"] - -from abc import abstractmethod - -import numpy as np -from scipy.stats import laplace, laplace_asymmetric -from traitlets import List - - -class PSFModel: - def __init__(self, **kwargs): - """ - Base component to describe image distortion due to the optics of the different cameras. - """ - - @classmethod - def from_name(cls, name, **kwargs): - """ - Obtain an instance of a subclass via its name - - Parameters - ---------- - name : str - Name of the subclass to obtain - - Returns - ------- - Instance - Instance of subclass to this class - """ - requested_subclass = cls.non_abstract_subclasses()[name] - return requested_subclass(**kwargs) - - @abstractmethod - def pdf(self, *args): - pass - - @abstractmethod - def update_model_parameters(self, *args): - pass - - -class ComaModel(PSFModel): - """ - PSF model, describing pure coma aberrations PSF effect - """ - - asymmetry_params = List( - default_value=[0.49244797, 9.23573115, 0.15216096], - help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", - ).tag(config=True) - radial_scale_params = List( - default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], - help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", - ).tag(config=True) - az_scale_params = List( - default_value=[0.24271557, 7.5511501, 0.02037972], - help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", - ).tag(config=True) - - def k_func(self, x): - return ( - 1 - - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) - - self.asymmetry_params[2] * x - ) - - def sr_func(self, x): - return ( - self.radial_scale_params[0] - - self.radial_scale_params[1] * x - + self.radial_scale_params[2] * x**2 - - self.radial_scale_params[3] * x**3 - ) - - def sf_func(self, x): - return self.az_scale_params[0] * np.exp( - -self.az_scale_params[1] * x - ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) - - def pdf(self, r, f): - return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( - f, *self.azimuthal_pdf_params - ) - - def update_model_parameters(self, r, f): - k = self.k_func(r) - sr = self.sr_func(r) - sf = self.sf_func(r) - self.radial_pdf_params = (k, r, sr) - self.azimuthal_pdf_params = (f, sf) diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py index 84064cbc1a3..658fe5291b4 100644 --- a/src/ctapipe/monitoring/interpolation.py +++ b/src/ctapipe/monitoring/interpolation.py @@ -9,11 +9,7 @@ from ctapipe.core import Component, traits -__all__ = [ - "Interpolator", - "PointingInterpolator", -] - +from .astropy_helpers import read_table class Interpolator(Component, metaclass=ABCMeta): """ diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py index 782aeae7435..02f4c4ce306 100644 --- a/src/ctapipe/monitoring/tests/test_interpolator.py +++ b/src/ctapipe/monitoring/tests/test_interpolator.py @@ -5,7 +5,9 @@ from astropy.table import Table from astropy.time import Time -from ctapipe.monitoring.interpolation import PointingInterpolator +from ctapipe.io.interpolation import ( + PointingInterpolator, +) t0 = Time("2022-01-01T00:00:00") From 9e19cb775caca6bd3d16998258dce3b1b718b590 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 23 Aug 2024 18:25:44 +0200 Subject: [PATCH 180/221] implement the StatisticsCalculator --- src/ctapipe/calib/camera/calibrator.py | 192 ----------- src/ctapipe/io/__init__.py | 2 + src/ctapipe/monitoring/calculator.py | 335 +++++++++++++++++++ src/ctapipe/monitoring/outlier.py | 12 +- src/ctapipe/monitoring/tests/test_outlier.py | 4 +- 5 files changed, 345 insertions(+), 200 deletions(-) create mode 100644 src/ctapipe/monitoring/calculator.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 07cd295d06c..12323587550 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -28,8 +28,6 @@ from ctapipe.io import TableLoader __all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", "CameraCalibrator", ] @@ -59,196 +57,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index cfb5fc5501e..ad20490a887 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,6 +18,8 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter +from .interpolation import Interpolator + __all__ = [ "HDF5TableWriter", "HDF5TableReader", diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py new file mode 100644 index 00000000000..26b5dfd6377 --- /dev/null +++ b/src/ctapipe/monitoring/calculator.py @@ -0,0 +1,335 @@ +""" +Definition of the ``CalibrationCalculator`` classes, providing all steps needed to +calculate the montoring data for the camera calibration. +""" + +import pathlib +from abc import abstractmethod + +import numpy as np +from astropy.table import vstack + +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + Bool, + CaselessStrEnum, + ComponentName, + Dict, + Float, + Int, + List, + Path, + TelescopeParameter, +) +from ctapipe.io import write_table +from ctapipe.io.tableloader import TableLoader +from ctapipe.monitoring.aggregator import StatisticsAggregator +from ctapipe.monitoring.outlier import OutlierDetector + +__all__ = [ + "CalibrationCalculator", + "StatisticsCalculator", +] + +PEDESTAL_GROUP = "/dl0/monitoring/telescope/pedestal" +FLATFIELD_GROUP = "/dl0/monitoring/telescope/flatfield" +TIMECALIB_GROUP = "/dl0/monitoring/telescope/time_calibration" + + +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_aggregator: str + The name of the StatisticsAggregator subclass to be used to aggregate the statistics + """ + + stats_aggregator_type = TelescopeParameter( + trait=ComponentName( + StatisticsAggregator, default_value="SigmaClippingAggregator" + ), + default_value="SigmaClippingAggregator", + help="Name of the StatisticsAggregator subclass to be used.", + ).tag(config=True) + + outlier_detector_type = List( + trait=Dict, + default_value=None, + allow_none=True, + help=( + "List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector." + ), + ).tag(config=True) + + calibration_type = CaselessStrEnum( + ["pedestal", "flatfield", "time_calibration"], + allow_none=False, + help="Set type of calibration which is needed to properly store the monitoring data", + ).tag(config=True) + + output_path = Path( + help="output filename", default_value=pathlib.Path("monitoring.camcalib.h5") + ).tag(config=True) + + overwrite = Bool(help="overwrite output file if it exists").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_aggregator=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_aggregator: ctapipe.monitoring.aggregator.StatisticsAggregator + The StatisticsAggregator to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.group = { + "pedestal": PEDESTAL_GROUP, + "flatfield": FLATFIELD_GROUP, + "time_calibration": TIMECALIB_GROUP, + } + + # Initialize the instances of StatisticsAggregator + self.stats_aggregator = {} + if stats_aggregator is None: + for _, _, name in self.stats_aggregator_type: + self.stats_aggregator[name] = StatisticsAggregator.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_aggregator.__class__.__name__ + self.stats_aggregator_type = [("type", "*", name)] + self.stats_aggregator[name] = stats_aggregator + + # Initialize the instances of OutlierDetector + self.outlier_detectors = {} + if self.outlier_detector_type is not None: + for outlier_detector in self.outlier_detector_type: + self.outlier_detectors[outlier_detector["apply_to"]] = ( + OutlierDetector.from_name( + name=outlier_detector["name"], + validity_range=outlier_detector["validity_range"], + subarray=self.subarray, + parent=self, + ) + ) + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class StatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + chunk_shift = Int( + default_value=None, + allow_none=True, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + two_pass = Bool(default_value=False, help="overwrite output file if it exists").tag( + config=True + ) + + faulty_pixels_threshold = Float( + default_value=0.1, + allow_none=True, + help=( + "Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk" + ), + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + + # Check if the chunk_shift is set for two pass mode + if self.two_pass and self.chunk_shift is None: + raise ValueError("chunk_shift must be set for two pass mode") + + # Get the aggregator + aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] + # Pass through the whole provided dl1 data + if self.two_pass: + self.aggregated_stats = aggregator( + table=dl1_table[tel_id], col_name=col_name, chunk_shift=None + ) + else: + self.aggregated_stats = aggregator( + table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift + ) + # Detect faulty pixels with mutiple instances of OutlierDetector + outlier_mask = np.zeros_like(self.aggregated_stats[0]["mean"], dtype=bool) + for aggregated_val, outlier_detector in self.outlier_detectors.items(): + outlier_mask = np.logical_or( + outlier_mask, + outlier_detector(self.aggregated_stats[aggregated_val]), + ) + # Add the outlier mask to the aggregated statistics + self.aggregated_stats["outlier_mask"] = outlier_mask + + if self.two_pass: + # Check if the camera has two gain channels + if outlier_mask.shape[1] == 2: + # Combine the outlier mask of both gain channels + outlier_mask = np.logical_or( + outlier_mask[:, 0, :], + outlier_mask[:, 1, :], + ) + # Calculate the fraction of faulty pixels over the camera + faulty_pixels_percentage = ( + np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] + ) + + # Check for faulty chunks if the threshold is exceeded + faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold + if np.any(faulty_chunks): + faulty_chunks_indices = np.where(faulty_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunks ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={self.aggregated_stats['time_start'][index]}; time_end={self.aggregated_stats['time_end'][index]}" + ) + + # Slice the dl1 table according to the previously caluclated start and end. + slice_start, slice_end = self._get_slice_range( + chunk_index=index, + faulty_previous_chunk=(index-1 in faulty_chunks_indices), + dl1_table_length=len(dl1_table[tel_id]) - 1, + ) + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] + + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + aggregated_stats_secondpass = aggregator( + table=dl1_table_sliced, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) + + # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass + outlier_mask = np.zeros_like(aggregated_stats_secondpass[0]["mean"], dtype=bool) + for aggregated_val, outlier_detector in self.outlier_detectors.items(): + outlier_mask = np.logical_or( + outlier_mask, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), + ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask + + # Stack the aggregated statistics of the second pass to the first pass + self.aggregated_stats = vstack([self.aggregated_stats, aggregated_stats_secondpass]) + # Sort the aggregated statistics based on the starting time + self.aggregated_stats.sort(["time_start"]) + else: + self.log.info( + "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + ) + + # Write the aggregated statistics and their outlier mask to the output file + write_table( + self.aggregated_stats, + self.output_path, + f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", + overwrite=self.overwrite, + ) + + def _get_slice_range( + self, + chunk_index, + faulty_previous_chunk, + dl1_table_length, + ) -> (int, int): + """ + Calculate the start and end indices for slicing the DL1 table to be used for the second pass. + + Parameters + ---------- + chunk_index : int + The index of the current faulty chunk being processed. + faulty_previous_chunk : bool + A flag indicating if the previous chunk was faulty. + dl1_table_length : int + The total length of the DL1 table. + + Returns + ------- + tuple + A tuple containing the start and end indices for slicing the DL1 table. + """ + + # Set the start of the slice to the first element of the dl1 table + slice_start = 0 + if chunk_index > 0: + # Get the start of the previous chunk + if faulty_previous_chunk: + slice_start = np.sum(self.aggregated_stats["n_events"][:chunk_index]) + else: + slice_start = np.sum( + self.aggregated_stats["n_events"][: chunk_index - 1] + ) + + # Set the end of the slice to the last element of the dl1 table + slice_end = dl1_table_length + if chunk_index < len(self.aggregated_stats) - 1: + # Get the stop of the next chunk + slice_end = np.sum(self.aggregated_stats["n_events"][: chunk_index + 2]) + + # Shift the start and end of the slice by the chunk_shift + slice_start += self.chunk_shift + slice_end -= self.chunk_shift - 1 + + return int(slice_start), int(slice_end) diff --git a/src/ctapipe/monitoring/outlier.py b/src/ctapipe/monitoring/outlier.py index 4ac387bbcd3..b2b84711cde 100644 --- a/src/ctapipe/monitoring/outlier.py +++ b/src/ctapipe/monitoring/outlier.py @@ -81,7 +81,7 @@ class MedianOutlierDetector(OutlierDetector): the configurable factors and the camera median of the statistic values. """ - median_range_factors = List( + validity_range = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -98,8 +98,8 @@ def __call__(self, column): # Detect outliers based on the deviation of the median distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.median_range_factors[0] * camera_median[:, :, np.newaxis], - deviation > self.median_range_factors[1] * camera_median[:, :, np.newaxis], + deviation < self.validity_range[0] * camera_median[:, :, np.newaxis], + deviation > self.validity_range[1] * camera_median[:, :, np.newaxis], ) return outliers @@ -112,7 +112,7 @@ class StdOutlierDetector(OutlierDetector): the configurable factors and the camera standard deviation of the statistic values. """ - std_range_factors = List( + validity_range = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -131,7 +131,7 @@ def __call__(self, column): # Detect outliers based on the deviation of the standard deviation distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.std_range_factors[0] * camera_std[:, :, np.newaxis], - deviation > self.std_range_factors[1] * camera_std[:, :, np.newaxis], + deviation < self.validity_range[0] * camera_std[:, :, np.newaxis], + deviation > self.validity_range[1] * camera_std[:, :, np.newaxis], ) return outliers diff --git a/src/ctapipe/monitoring/tests/test_outlier.py b/src/ctapipe/monitoring/tests/test_outlier.py index da7d7619b33..61f1d8cb91d 100644 --- a/src/ctapipe/monitoring/tests/test_outlier.py +++ b/src/ctapipe/monitoring/tests/test_outlier.py @@ -56,7 +56,7 @@ def test_median_detection(example_subarray): # In this test, the interval [-0.9, 8] corresponds to multiplication factors # typical used for the median values of charge images of flat-field events detector = MedianOutlierDetector( - subarray=example_subarray, median_range_factors=[-0.9, 8.0] + subarray=example_subarray, validity_range=[-0.9, 8.0] ) # Detect outliers outliers = detector(table["median"]) @@ -89,7 +89,7 @@ def test_std_detection(example_subarray): # typical used for the std values of charge images of flat-field events # and median (and std) values of charge images of pedestal events detector = StdOutlierDetector( - subarray=example_subarray, std_range_factors=[-15.0, 15.0] + subarray=example_subarray, validity_range=[-15.0, 15.0] ) ff_outliers = detector(ff_table["std"]) ped_outliers = detector(ped_table["median"]) From 9540f21415d18bf9ef5c67ab2925a5228997af0e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sat, 24 Aug 2024 11:19:38 +0200 Subject: [PATCH 181/221] removed the helper function to get the start and end slices Since agregation is chunk has always the same n_events, we can simplify the retrieving of the start and end slices. Therefore we do not need a helper function anymore --- src/ctapipe/monitoring/calculator.py | 132 +++++++++++---------------- 1 file changed, 53 insertions(+), 79 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 26b5dfd6377..56de565c567 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -159,18 +159,19 @@ class StatisticsCalculator(CalibrationCalculator): chunk_shift = Int( default_value=None, allow_none=True, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + help="Number of samples to shift the aggregation chunk for the calculation of the statistical values", ).tag(config=True) - two_pass = Bool(default_value=False, help="overwrite output file if it exists").tag( - config=True - ) + second_pass = Bool( + default_value=False, help="overwrite output file if it exists" + ).tag(config=True) faulty_pixels_threshold = Float( - default_value=0.1, + default_value=10.0, allow_none=True, help=( - "Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk" + "Threshold in percentage of faulty pixels over the camera " + "to conduct second pass with a refined shift of the chunk." ), ).tag(config=True) @@ -197,31 +198,33 @@ def __call__( ) # Check if the chunk_shift is set for two pass mode - if self.two_pass and self.chunk_shift is None: - raise ValueError("chunk_shift must be set for two pass mode") + if self.second_pass and self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is selected" + ) # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 data - if self.two_pass: - self.aggregated_stats = aggregator( + if self.second_pass: + aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=None ) else: - self.aggregated_stats = aggregator( + aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift ) # Detect faulty pixels with mutiple instances of OutlierDetector - outlier_mask = np.zeros_like(self.aggregated_stats[0]["mean"], dtype=bool) + outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( outlier_mask, - outlier_detector(self.aggregated_stats[aggregated_val]), + outlier_detector(aggregated_stats[aggregated_val]), ) # Add the outlier mask to the aggregated statistics - self.aggregated_stats["outlier_mask"] = outlier_mask + aggregated_stats["outlier_mask"] = outlier_mask - if self.two_pass: + if self.second_pass: # Check if the camera has two gain channels if outlier_mask.shape[1] == 2: # Combine the outlier mask of both gain channels @@ -232,24 +235,33 @@ def __call__( # Calculate the fraction of faulty pixels over the camera faulty_pixels_percentage = ( np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] - ) + ) * 100.0 # Check for faulty chunks if the threshold is exceeded faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold if np.any(faulty_chunks): + chunk_size = aggregated_stats["n_events"][0] faulty_chunks_indices = np.where(faulty_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.warning( - f"Faulty chunks ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={self.aggregated_stats['time_start'][index]}; time_end={self.aggregated_stats['time_end'][index]}" + f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - - # Slice the dl1 table according to the previously caluclated start and end. - slice_start, slice_end = self._get_slice_range( - chunk_index=index, - faulty_previous_chunk=(index-1 in faulty_chunks_indices), - dl1_table_length=len(dl1_table[tel_id]) - 1, + # Calculate the start of the slice based + slice_start = ( + chunk_size * index + if index - 1 in faulty_chunks_indices + else chunk_size * (index - 1) ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min( + len(dl1_table[tel_id]) - 1, chunk_size * (index + 2) + ) - (self.chunk_shift - 1) + # Slice the dl1 table according to the previously caluclated start and end. dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift @@ -261,19 +273,28 @@ def __call__( ) # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass - outlier_mask = np.zeros_like(aggregated_stats_secondpass[0]["mean"], dtype=bool) - for aggregated_val, outlier_detector in self.outlier_detectors.items(): - outlier_mask = np.logical_or( - outlier_mask, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass[0]["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector( + aggregated_stats_secondpass[aggregated_val] + ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass # Stack the aggregated statistics of the second pass to the first pass - self.aggregated_stats = vstack([self.aggregated_stats, aggregated_stats_secondpass]) + aggregated_stats = vstack( + [aggregated_stats, aggregated_stats_secondpass] + ) # Sort the aggregated statistics based on the starting time - self.aggregated_stats.sort(["time_start"]) + aggregated_stats.sort(["time_start"]) else: self.log.info( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." @@ -281,55 +302,8 @@ def __call__( # Write the aggregated statistics and their outlier mask to the output file write_table( - self.aggregated_stats, + aggregated_stats, self.output_path, f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", overwrite=self.overwrite, ) - - def _get_slice_range( - self, - chunk_index, - faulty_previous_chunk, - dl1_table_length, - ) -> (int, int): - """ - Calculate the start and end indices for slicing the DL1 table to be used for the second pass. - - Parameters - ---------- - chunk_index : int - The index of the current faulty chunk being processed. - faulty_previous_chunk : bool - A flag indicating if the previous chunk was faulty. - dl1_table_length : int - The total length of the DL1 table. - - Returns - ------- - tuple - A tuple containing the start and end indices for slicing the DL1 table. - """ - - # Set the start of the slice to the first element of the dl1 table - slice_start = 0 - if chunk_index > 0: - # Get the start of the previous chunk - if faulty_previous_chunk: - slice_start = np.sum(self.aggregated_stats["n_events"][:chunk_index]) - else: - slice_start = np.sum( - self.aggregated_stats["n_events"][: chunk_index - 1] - ) - - # Set the end of the slice to the last element of the dl1 table - slice_end = dl1_table_length - if chunk_index < len(self.aggregated_stats) - 1: - # Get the stop of the next chunk - slice_end = np.sum(self.aggregated_stats["n_events"][: chunk_index + 2]) - - # Shift the start and end of the slice by the chunk_shift - slice_start += self.chunk_shift - slice_end -= self.chunk_shift - 1 - - return int(slice_start), int(slice_end) From 7a2c77e712235b474f2ee3aa8a218053798dd06d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:01:41 +0200 Subject: [PATCH 182/221] polish docstrings --- CODEOWNERS | 2 + src/ctapipe/monitoring/calculator.py | 70 +++++++++++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 06c29bc6c78..70c53117a07 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,6 +5,8 @@ ctapipe/calib/camera @watsonjj ctapipe/image/extractor.py @watsonjj @HealthyPear +ctapipe/monitoring @TjarkMiener + ctapipe/reco/HillasReconstructor.py @HealthyPear ctapipe/reco/tests/test_HillasReconstructor.py @HealthyPear diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 56de565c567..efccfb11368 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -38,12 +38,25 @@ class CalibrationCalculator(TelescopeComponent): """ - Base component for various calibration calculators + Base component for calibration calculators. + + This class provides the foundational methods and attributes for + calculating camera-related monitoring data. It is designed + to be extended by specific calibration calculators that implement + the required methods for different types of calibration. Attributes ---------- - stats_aggregator: str - The name of the StatisticsAggregator subclass to be used to aggregate the statistics + stats_aggregator_type : ctapipe.core.traits.TelescopeParameter + The type of StatisticsAggregator to be used for aggregating statistics. + outlier_detector_list : list of dict + List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. + calibration_type : ctapipe.core.traits.CaselessStrEnum + The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. + output_path : ctapipe.core.traits.Path + The output filename where the calibration data will be stored. + overwrite : ctapipe.core.traits.Bool + Whether to overwrite the output file if it exists. """ stats_aggregator_type = TelescopeParameter( @@ -54,12 +67,14 @@ class CalibrationCalculator(TelescopeComponent): help="Name of the StatisticsAggregator subclass to be used.", ).tag(config=True) - outlier_detector_type = List( + outlier_detector_list = List( trait=Dict, default_value=None, allow_none=True, help=( - "List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector." + "List of dicts containing the name of the OutlierDetector subclass to be used, " + "the aggregated value to which the detector should be applied, " + "and the validity range of the detector." ), ).tag(config=True) @@ -70,10 +85,10 @@ class CalibrationCalculator(TelescopeComponent): ).tag(config=True) output_path = Path( - help="output filename", default_value=pathlib.Path("monitoring.camcalib.h5") + help="Output filename", default_value=pathlib.Path("monitoring.camcalib.h5") ).tag(config=True) - overwrite = Bool(help="overwrite output file if it exists").tag(config=True) + overwrite = Bool(help="Overwrite output file if it exists").tag(config=True) def __init__( self, @@ -98,7 +113,7 @@ def __init__( Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` stats_aggregator: ctapipe.monitoring.aggregator.StatisticsAggregator - The StatisticsAggregator to use. If None, the default via the + The ``StatisticsAggregator`` to use. If None, the default via the configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @@ -124,8 +139,8 @@ def __init__( # Initialize the instances of OutlierDetector self.outlier_detectors = {} - if self.outlier_detector_type is not None: - for outlier_detector in self.outlier_detector_type: + if self.outlier_detector_list is not None: + for outlier_detector in self.outlier_detector_list: self.outlier_detectors[outlier_detector["apply_to"]] = ( OutlierDetector.from_name( name=outlier_detector["name"], @@ -136,7 +151,7 @@ def __init__( ) @abstractmethod - def __call__(self, input_url, tel_id): + def __call__(self, input_url, tel_id, col_name): """ Call the relevant functions to calculate the calibration coefficients for a given set of events @@ -154,16 +169,35 @@ def __call__(self, input_url, tel_id): class StatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. + + This class inherits from CalibrationCalculator and is responsible for + calculating various statistics from calibration events, such as pedestal + and flat-field data. It reads the data, aggregates statistics, detects + outliers, handles faulty data chunks, and stores the monitoring data. + The default option is to conduct only one pass over the data with non-overlapping + chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. + Two passes over the data, set via the ``second_pass``-flag, can be conducted + with a refined shift of the chunk in regions of trouble with a high percentage + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( default_value=None, allow_none=True, - help="Number of samples to shift the aggregation chunk for the calculation of the statistical values", + help=( + "Number of samples to shift the aggregation chunk for the " + "calculation of the statistical values. If second_pass is set, " + "the first pass is conducted without overlapping chunks (chunk_shift=None) " + "and the second pass with a refined shift of the chunk in regions of trouble." + ), ).tag(config=True) second_pass = Bool( - default_value=False, help="overwrite output file if it exists" + default_value=False, + help=( + "Set whether to conduct a second pass over the data " + "with a refined shift of the chunk in regions of trouble." + ), ).tag(config=True) faulty_pixels_threshold = Float( @@ -171,7 +205,8 @@ class StatisticsCalculator(CalibrationCalculator): allow_none=True, help=( "Threshold in percentage of faulty pixels over the camera " - "to conduct second pass with a refined shift of the chunk." + "to conduct second pass with a refined shift of the chunk " + "in regions of trouble." ), ).tag(config=True) @@ -182,7 +217,7 @@ def __call__( col_name="image", ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( telescopes=tel_id, @@ -197,7 +232,7 @@ def __call__( pointing=False, ) - # Check if the chunk_shift is set for two pass mode + # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: raise ValueError( "chunk_shift must be set if second pass over the data is selected" @@ -214,7 +249,7 @@ def __call__( aggregated_stats = aggregator( table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift ) - # Detect faulty pixels with mutiple instances of OutlierDetector + # Detect faulty pixels with mutiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( @@ -224,6 +259,7 @@ def __call__( # Add the outlier mask to the aggregated statistics aggregated_stats["outlier_mask"] = outlier_mask + # Conduct a second pass over the data if self.second_pass: # Check if the camera has two gain channels if outlier_mask.shape[1] == 2: From 9afa2773733f40f44aca569cd98c031f348c6ed5 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:13:27 +0200 Subject: [PATCH 183/221] further polishing of docstrings --- src/ctapipe/monitoring/calculator.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index efccfb11368..9d948bde388 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -54,7 +54,7 @@ class CalibrationCalculator(TelescopeComponent): calibration_type : ctapipe.core.traits.CaselessStrEnum The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. output_path : ctapipe.core.traits.Path - The output filename where the calibration data will be stored. + The output filename where the monitoring data will be stored. overwrite : ctapipe.core.traits.Bool Whether to overwrite the output file if it exists. """ @@ -153,16 +153,21 @@ def __init__( @abstractmethod def __call__(self, input_url, tel_id, col_name): """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events + Calculate the monitoring data for a given set of events. + + This method should be implemented by subclasses to perform the specific + calibration calculations required for different types of calibration. Parameters ---------- input_url : str - URL where the events are stored from which the calibration coefficients + URL where the events are stored from which the monitoring data are to be calculated tel_id : int - The telescope id + The telescope ID for which the calibration is being performed. + col_name : str + The name of the column in the data from which the statistics + will be aggregated. """ @@ -283,7 +288,7 @@ def __call__( self.log.warning( f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - # Calculate the start of the slice based + # Calculate the start of the slice based weather the previous chunk was faulty or not slice_start = ( chunk_size * index if index - 1 in faulty_chunks_indices From fc3c869db90368a238c3a34e6ffcbc847956332f Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:17:28 +0200 Subject: [PATCH 184/221] fix typo --- src/ctapipe/monitoring/calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 9d948bde388..b6fdd3e0417 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -288,7 +288,7 @@ def __call__( self.log.warning( f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) - # Calculate the start of the slice based weather the previous chunk was faulty or not + # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( chunk_size * index if index - 1 in faulty_chunks_indices From 2f7466a0d1254fefede0bed0ddacf11d02cb6e2e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 15:20:38 +0200 Subject: [PATCH 185/221] move check of config settings before loading of data --- src/ctapipe/monitoring/calculator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index b6fdd3e0417..09bb1a0ab01 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -222,6 +222,12 @@ def __call__( col_name="image", ): + # Check if the chunk_shift is set for second pass mode + if self.second_pass and self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is selected" + ) + # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` input_data = TableLoader(input_url=input_url) dl1_table = input_data.read_telescope_events_by_id( @@ -237,12 +243,6 @@ def __call__( pointing=False, ) - # Check if the chunk_shift is set for second pass mode - if self.second_pass and self.chunk_shift is None: - raise ValueError( - "chunk_shift must be set if second pass over the data is selected" - ) - # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 data From 534cbe8b8ee2fc80e068a801fc913a9f52de0c9c Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 16:04:31 +0200 Subject: [PATCH 186/221] moved Interpolator outside This branch should only host the devs for the stats calculator --- src/ctapipe/calib/camera/calibrator.py | 11 +-- src/ctapipe/calib/camera/extractor.py | 4 - .../calib/camera/tests/test_extractors.py | 84 ------------------- src/ctapipe/io/__init__.py | 3 +- src/ctapipe/monitoring/calculator.py | 2 +- 5 files changed, 4 insertions(+), 100 deletions(-) delete mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 12323587550..853ba3f7da8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -3,33 +3,24 @@ calibration and image extraction, as well as supporting algorithms. """ -import pickle -from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Float, - Int, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import TableLoader -__all__ = [ - "CameraCalibrator", -] +__all__ = ["CameraCalibrator"] @cache diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 7699f8cc83d..3075b33a9ca 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -1,9 +1,5 @@ """ -<<<<<<< HEAD -Extraction algorithms to compute the statistics from a chunk of images -======= Extraction algorithms to compute the statistics from a sequence of images ->>>>>>> f7d3223e (solved merge conflicts) """ __all__ = [ diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py deleted file mode 100644 index a83c93fd1c0..00000000000 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Tests for StatisticsExtractor and related functions -""" - -import numpy as np -import pytest -from astropy.table import QTable - -from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor - - -@pytest.fixture(name="test_plainextractor") -def fixture_test_plainextractor(example_subarray): - """test the PlainExtractor""" - return PlainExtractor(subarray=example_subarray, chunk_size=2500) - - -@pytest.fixture(name="test_sigmaclippingextractor") -def fixture_test_sigmaclippingextractor(example_subarray): - """test the SigmaClippingExtractor""" - return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) - - -def test_extractors(test_plainextractor, test_sigmaclippingextractor): - """test basic functionality of the StatisticsExtractors""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - - plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) - - -def test_check_outliers(test_sigmaclippingextractor): - """test detection ability of outliers""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_data[:, 0, 120] = 120.0 - flatfield_dl1_data[:, 1, 67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] - assert sigmaclipping_stats_list[0].median_outliers[1][67] - assert sigmaclipping_stats_list[1].median_outliers[0][120] - assert sigmaclipping_stats_list[1].median_outliers[1][67] - - -def test_check_chunk_shift(test_sigmaclippingextractor): - """test the chunk shift option and the boundary case for the last chunk""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, chunk_shift=2000 - ) - - # check if three chunks are used for the extraction - assert len(sigmaclipping_stats_list) == 3 diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index ad20490a887..229f212b766 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,7 +18,7 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter -from .interpolation import Interpolator +from .pointing import PointingInterpolator __all__ = [ "HDF5TableWriter", @@ -37,4 +37,5 @@ "DataWriter", "DATA_MODEL_VERSION", "get_hdf5_datalevels", + "PointingInterpolator", ] diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 09bb1a0ab01..dc56259425c 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -73,7 +73,7 @@ class CalibrationCalculator(TelescopeComponent): allow_none=True, help=( "List of dicts containing the name of the OutlierDetector subclass to be used, " - "the aggregated value to which the detector should be applied, " + "the aggregated statistic value to which the detector should be applied, " "and the validity range of the detector." ), ).tag(config=True) From 53c379be30df8f8fc6fba7d0aff525787e0bbc32 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 16:07:55 +0200 Subject: [PATCH 187/221] removed Interpolator artifacts --- src/ctapipe/io/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index 229f212b766..afe96f39430 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,7 +18,6 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter -from .pointing import PointingInterpolator __all__ = [ "HDF5TableWriter", @@ -37,5 +36,4 @@ "DataWriter", "DATA_MODEL_VERSION", "get_hdf5_datalevels", - "PointingInterpolator", ] From dfb179efca6683531649a44646922c081c72e8bf Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 18:34:04 +0200 Subject: [PATCH 188/221] removed reading part with TableLoader reading should be done in the tool outside of this component --- src/ctapipe/monitoring/calculator.py | 67 +++++++++++++--------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index dc56259425c..f93a88e0b78 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -22,7 +22,6 @@ TelescopeParameter, ) from ctapipe.io import write_table -from ctapipe.io.tableloader import TableLoader from ctapipe.monitoring.aggregator import StatisticsAggregator from ctapipe.monitoring.outlier import OutlierDetector @@ -151,7 +150,7 @@ def __init__( ) @abstractmethod - def __call__(self, input_url, tel_id, col_name): + def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): """ Calculate the monitoring data for a given set of events. @@ -160,14 +159,15 @@ def __call__(self, input_url, tel_id, col_name): Parameters ---------- - input_url : str - URL where the events are stored from which the monitoring data - are to be calculated + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix) + and timestamps of shape (n_images, ) + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing tel_id : int - The telescope ID for which the calibration is being performed. + Telescope ID for which the calibration is being performed col_name : str - The name of the column in the data from which the statistics - will be aggregated. + Column name in the table from which the statistics will be aggregated """ @@ -177,13 +177,13 @@ class StatisticsCalculator(CalibrationCalculator): This class inherits from CalibrationCalculator and is responsible for calculating various statistics from calibration events, such as pedestal - and flat-field data. It reads the data, aggregates statistics, detects - outliers, handles faulty data chunks, and stores the monitoring data. + and flat-field data. It aggregates statistics, detects outliers, + handles faulty data chunks, and stores the monitoring data. The default option is to conduct only one pass over the data with non-overlapping chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. Two passes over the data, set via the ``second_pass``-flag, can be conducted with a refined shift of the chunk in regions of trouble with a high percentage - of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( @@ -217,7 +217,8 @@ class StatisticsCalculator(CalibrationCalculator): def __call__( self, - input_url, + table, + masked_pixels_of_sample, tel_id, col_name="image", ): @@ -228,31 +229,22 @@ def __call__( "chunk_shift must be set if second pass over the data is selected" ) - # Read the whole dl1-like images of pedestal and flat-field data with the ``TableLoader`` - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] - # Pass through the whole provided dl1 data + # Pass through the whole provided dl1 table if self.second_pass: aggregated_stats = aggregator( - table=dl1_table[tel_id], col_name=col_name, chunk_shift=None + table=table[tel_id], + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=None, ) else: aggregated_stats = aggregator( - table=dl1_table[tel_id], col_name=col_name, chunk_shift=self.chunk_shift + table=table[tel_id], + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) # Detect faulty pixels with mutiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) @@ -299,16 +291,17 @@ def __call__( slice_start = max(0, slice_start) + self.chunk_shift # Set the end of the slice to the last element of the dl1 table if out of bound # and subtract one ``chunk_shift``. - slice_end = min( - len(dl1_table[tel_id]) - 1, chunk_size * (index + 2) - ) - (self.chunk_shift - 1) + slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) # Slice the dl1 table according to the previously caluclated start and end. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_end] + table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. aggregated_stats_secondpass = aggregator( - table=dl1_table_sliced, + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=self.chunk_shift, ) @@ -328,7 +321,9 @@ def __call__( ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["outlier_mask"] = ( + outlier_mask_secondpass + ) # Stack the aggregated statistics of the second pass to the first pass aggregated_stats = vstack( From 955ff89c8ae60e2c112bcb65cc8255b82128a2aa Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sun, 25 Aug 2024 18:55:03 +0200 Subject: [PATCH 189/221] removed writing part writing should be also done in the tool outside of this component --- src/ctapipe/monitoring/calculator.py | 55 ++++++---------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index f93a88e0b78..d3462aafb6a 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -3,25 +3,21 @@ calculate the montoring data for the camera calibration. """ -import pathlib from abc import abstractmethod import numpy as np -from astropy.table import vstack +from astropy.table import Table, vstack from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Bool, - CaselessStrEnum, ComponentName, Dict, Float, Int, List, - Path, TelescopeParameter, ) -from ctapipe.io import write_table from ctapipe.monitoring.aggregator import StatisticsAggregator from ctapipe.monitoring.outlier import OutlierDetector @@ -30,10 +26,6 @@ "StatisticsCalculator", ] -PEDESTAL_GROUP = "/dl0/monitoring/telescope/pedestal" -FLATFIELD_GROUP = "/dl0/monitoring/telescope/flatfield" -TIMECALIB_GROUP = "/dl0/monitoring/telescope/time_calibration" - class CalibrationCalculator(TelescopeComponent): """ @@ -50,12 +42,6 @@ class CalibrationCalculator(TelescopeComponent): The type of StatisticsAggregator to be used for aggregating statistics. outlier_detector_list : list of dict List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. - calibration_type : ctapipe.core.traits.CaselessStrEnum - The type of calibration (e.g., pedestal, flatfield, time_calibration) which is needed to properly store the monitoring data. - output_path : ctapipe.core.traits.Path - The output filename where the monitoring data will be stored. - overwrite : ctapipe.core.traits.Bool - Whether to overwrite the output file if it exists. """ stats_aggregator_type = TelescopeParameter( @@ -77,18 +63,6 @@ class CalibrationCalculator(TelescopeComponent): ), ).tag(config=True) - calibration_type = CaselessStrEnum( - ["pedestal", "flatfield", "time_calibration"], - allow_none=False, - help="Set type of calibration which is needed to properly store the monitoring data", - ).tag(config=True) - - output_path = Path( - help="Output filename", default_value=pathlib.Path("monitoring.camcalib.h5") - ).tag(config=True) - - overwrite = Bool(help="Overwrite output file if it exists").tag(config=True) - def __init__( self, subarray, @@ -118,12 +92,6 @@ def __init__( super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray - self.group = { - "pedestal": PEDESTAL_GROUP, - "flatfield": FLATFIELD_GROUP, - "time_calibration": TIMECALIB_GROUP, - } - # Initialize the instances of StatisticsAggregator self.stats_aggregator = {} if stats_aggregator is None: @@ -150,7 +118,7 @@ def __init__( ) @abstractmethod - def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): + def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: """ Calculate the monitoring data for a given set of events. @@ -168,6 +136,11 @@ def __call__(self, table, masked_pixels_of_sample, tel_id, col_name): Telescope ID for which the calibration is being performed col_name : str Column name in the table from which the statistics will be aggregated + + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics and their outlier masks """ @@ -178,7 +151,7 @@ class StatisticsCalculator(CalibrationCalculator): This class inherits from CalibrationCalculator and is responsible for calculating various statistics from calibration events, such as pedestal and flat-field data. It aggregates statistics, detects outliers, - handles faulty data chunks, and stores the monitoring data. + handles faulty data chunks. The default option is to conduct only one pass over the data with non-overlapping chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. Two passes over the data, set via the ``second_pass``-flag, can be conducted @@ -221,7 +194,7 @@ def __call__( masked_pixels_of_sample, tel_id, col_name="image", - ): + ) -> Table: # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: @@ -335,11 +308,5 @@ def __call__( self.log.info( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." ) - - # Write the aggregated statistics and their outlier mask to the output file - write_table( - aggregated_stats, - self.output_path, - f"{self.group[self.calibration_type]}/tel_{tel_id:03d}", - overwrite=self.overwrite, - ) + # Return the aggregated statistics and their outlier masks + return aggregated_stats \ No newline at end of file From 47fc85dae76c2759c6e1cee70a073ed59b6b4726 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 10:02:30 +0200 Subject: [PATCH 190/221] add unit tests for calculators --- src/ctapipe/monitoring/aggregator.py | 4 +- src/ctapipe/monitoring/calculator.py | 46 ++++---- .../monitoring/tests/test_calculator.py | 106 ++++++++++++++++++ 3 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 src/ctapipe/monitoring/tests/test_calculator.py diff --git a/src/ctapipe/monitoring/aggregator.py b/src/ctapipe/monitoring/aggregator.py index ee8b5566c7c..9dc3aa485fd 100644 --- a/src/ctapipe/monitoring/aggregator.py +++ b/src/ctapipe/monitoring/aggregator.py @@ -65,8 +65,8 @@ def __call__( Parameters ---------- table : astropy.table.Table - table with images of shape (n_images, n_channels, n_pix) - and timestamps of shape (n_images, ) + table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) masked_pixels_of_sample : ndarray, optional boolean array of masked pixels of shape (n_pix, ) that are not available for processing chunk_shift : int, optional diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index d3462aafb6a..750e44cb75f 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -53,7 +53,7 @@ class CalibrationCalculator(TelescopeComponent): ).tag(config=True) outlier_detector_list = List( - trait=Dict, + trait=Dict(), default_value=None, allow_none=True, help=( @@ -118,7 +118,9 @@ def __init__( ) @abstractmethod - def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: + def __call__( + self, table, tel_id, masked_pixels_of_sample=None, col_name="image" + ) -> Table: """ Calculate the monitoring data for a given set of events. @@ -128,15 +130,15 @@ def __call__(self, table, masked_pixels_of_sample, tel_id, col_name) -> Table: Parameters ---------- table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix) - and timestamps of shape (n_images, ) - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) tel_id : int Telescope ID for which the calibration is being performed + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing col_name : str Column name in the table from which the statistics will be aggregated - + Returns ------- astropy.table.Table @@ -191,8 +193,8 @@ class StatisticsCalculator(CalibrationCalculator): def __call__( self, table, - masked_pixels_of_sample, tel_id, + masked_pixels_of_sample=None, col_name="image", ) -> Table: @@ -207,20 +209,21 @@ def __call__( # Pass through the whole provided dl1 table if self.second_pass: aggregated_stats = aggregator( - table=table[tel_id], + table=table, masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=None, ) else: aggregated_stats = aggregator( - table=table[tel_id], + table=table, masked_pixels_of_sample=masked_pixels_of_sample, col_name=col_name, chunk_shift=self.chunk_shift, ) + # Detect faulty pixels with mutiple instances of ``OutlierDetector`` - outlier_mask = np.zeros_like(aggregated_stats[0]["mean"], dtype=bool) + outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( outlier_mask, @@ -251,7 +254,7 @@ def __call__( for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.warning( - f"Faulty chunk ({int(faulty_pixels_percentage[index]*100.0)}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" + f"Faulty chunk ({int(faulty_pixels_percentage[index])}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( @@ -272,16 +275,19 @@ def __call__( # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) + # Checking for the length of the sliced table to be greater than he chunk_size + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass[0]["mean"], dtype=bool + aggregated_stats_secondpass["mean"], dtype=bool ) for ( aggregated_val, @@ -309,4 +315,4 @@ def __call__( "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." ) # Return the aggregated statistics and their outlier masks - return aggregated_stats \ No newline at end of file + return aggregated_stats diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py new file mode 100644 index 00000000000..2878e6d4a0f --- /dev/null +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -0,0 +1,106 @@ +""" +Tests for CalibrationCalculator and related functions +""" + +import numpy as np +from astropy.table import Table +from astropy.time import Time +from traitlets.config.loader import Config + +from ctapipe.monitoring.aggregator import PlainAggregator +from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator + + +def test_onepass_calculator(example_subarray): + """test basic 'one pass' functionality of the StatisticsCalculator""" + + # Create dummy data for testing + times = Time( + np.linspace(60117.911, 60117.9258, num=5000), scale="tai", format="mjd" + ) + event_ids = np.linspace(35, 725000, num=5000, dtype=int) + rng = np.random.default_rng(0) + charge_data = rng.normal(77.0, 10.0, size=(5000, 2, 1855)) + # Create tables + charge_table = Table( + [times, event_ids, charge_data], + names=("time_mono", "event_id", "image"), + ) + # Initialize the aggregators and calculators + chunk_size = 500 + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + calculator = CalibrationCalculator.from_name( + name="StatisticsCalculator", + subarray=example_subarray, + stats_aggregator=aggregator, + ) + calculator_chunk_shift = StatisticsCalculator( + subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=250 + ) + # Compute the statistical values + stats = calculator(table=charge_table, tel_id=1) + stats_chunk_shift = calculator_chunk_shift(table=charge_table, tel_id=1) + + # Check if the calculated statistical values are reasonable + # for a camera with two gain channels + np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) + np.testing.assert_allclose(stats[1]["median"], 77.0, atol=2.5) + np.testing.assert_allclose(stats[0]["std"], 10.0, atol=2.5) + # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows + assert len(stats) * 2 == len(stats_chunk_shift) + 1 + +def test_secondpass_calculator(example_subarray): + """test the chunk shift option and the boundary case for the last chunk""" + + # Create dummy data for testing + times = Time( + np.linspace(60117.911, 60117.9258, num=5500), scale="tai", format="mjd" + ) + event_ids = np.linspace(35, 725000, num=5500, dtype=int) + rng = np.random.default_rng(0) + ped_data = rng.normal(2.0, 5.0, size=(5500, 2, 1855)) + # Create table + ped_table = Table( + [times, event_ids, ped_data], + names=("time_mono", "event_id", "image"), + ) + # Create configuration + config = Config( + { + "StatisticsCalculator": { + "stats_aggregator_type": [ + ("id", 1, "SigmaClippingAggregator"), + ], + "outlier_detector_list": [ + { + "apply_to": "mean", + "name": "StdOutlierDetector", + "validity_range": [-2.0, 2.0], + }, + { + "apply_to": "median", + "name": "StdOutlierDetector", + "validity_range": [-3.0, 3.0], + }, + { + "apply_to": "std", + "name": "RangeOutlierDetector", + "validity_range": [2.0, 8.0], + }, + ], + "chunk_shift": 100, + "second_pass": True, + "faulty_pixels_threshold": 1.0, + }, + "SigmaClippingAggregator": { + "chunk_size": 500, + }, + } + ) + # Initialize the calculator from config + calculator = StatisticsCalculator(subarray=example_subarray, config=config) + # Compute aggregated statistic values + stats = calculator(ped_table, 1, col_name="image") + # Check if the second pass was activated + assert len(stats) > 20 + From 28101d176e550fdcca269c4282262821ce9d56ff Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 10:06:18 +0200 Subject: [PATCH 191/221] add unit tests for calculators --- src/ctapipe/monitoring/calculator.py | 27 +++++++++---------- .../monitoring/tests/test_calculator.py | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 750e44cb75f..5249ab49a14 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -108,13 +108,13 @@ def __init__( self.outlier_detectors = {} if self.outlier_detector_list is not None: for outlier_detector in self.outlier_detector_list: - self.outlier_detectors[outlier_detector["apply_to"]] = ( - OutlierDetector.from_name( - name=outlier_detector["name"], - validity_range=outlier_detector["validity_range"], - subarray=self.subarray, - parent=self, - ) + self.outlier_detectors[ + outlier_detector["apply_to"] + ] = OutlierDetector.from_name( + name=outlier_detector["name"], + validity_range=outlier_detector["validity_range"], + subarray=self.subarray, + parent=self, ) @abstractmethod @@ -197,7 +197,6 @@ def __call__( masked_pixels_of_sample=None, col_name="image", ) -> Table: - # Check if the chunk_shift is set for second pass mode if self.second_pass and self.chunk_shift is None: raise ValueError( @@ -222,7 +221,7 @@ def __call__( chunk_shift=self.chunk_shift, ) - # Detect faulty pixels with mutiple instances of ``OutlierDetector`` + # Detect faulty pixels with multiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): outlier_mask = np.logical_or( @@ -270,7 +269,7 @@ def __call__( slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( self.chunk_shift - 1 ) - # Slice the dl1 table according to the previously caluclated start and end. + # Slice the dl1 table according to the previously calculated start and end. table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift @@ -285,7 +284,7 @@ def __call__( chunk_shift=self.chunk_shift, ) - # Detect faulty pixels with mutiple instances of OutlierDetector of the second pass + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass outlier_mask_secondpass = np.zeros_like( aggregated_stats_secondpass["mean"], dtype=bool ) @@ -300,9 +299,9 @@ def __call__( ), ) # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = ( - outlier_mask_secondpass - ) + aggregated_stats_secondpass[ + "outlier_mask" + ] = outlier_mask_secondpass # Stack the aggregated statistics of the second pass to the first pass aggregated_stats = vstack( diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 2878e6d4a0f..08c818de0af 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -49,6 +49,7 @@ def test_onepass_calculator(example_subarray): # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows assert len(stats) * 2 == len(stats_chunk_shift) + 1 + def test_secondpass_calculator(example_subarray): """test the chunk shift option and the boundary case for the last chunk""" @@ -103,4 +104,3 @@ def test_secondpass_calculator(example_subarray): stats = calculator(ped_table, 1, col_name="image") # Check if the second pass was activated assert len(stats) > 20 - From 8b3d5285e0f5f8bbad9b0e826694744f9fd1ceeb Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 15:38:53 +0200 Subject: [PATCH 192/221] split __call__ function into two function for the first and second pass second pass also has an argument to pass the list of faulty/valid chunks which can be a logical_and from multiple first passes --- src/ctapipe/monitoring/calculator.py | 326 ++++++++++-------- .../monitoring/tests/test_calculator.py | 67 ++-- 2 files changed, 220 insertions(+), 173 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 5249ab49a14..9359e0b1689 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -3,14 +3,11 @@ calculate the montoring data for the camera calibration. """ -from abc import abstractmethod - import numpy as np -from astropy.table import Table, vstack +from astropy.table import Table from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( - Bool, ComponentName, Dict, Float, @@ -117,66 +114,30 @@ def __init__( parent=self, ) - @abstractmethod - def __call__( - self, table, tel_id, masked_pixels_of_sample=None, col_name="image" - ) -> Table: - """ - Calculate the monitoring data for a given set of events. - - This method should be implemented by subclasses to perform the specific - calibration calculations required for different types of calibration. - - Parameters - ---------- - table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and - timestamps of shape (n_images, ) - tel_id : int - Telescope ID for which the calibration is being performed - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing - col_name : str - Column name in the table from which the statistics will be aggregated - - Returns - ------- - astropy.table.Table - Table containing the aggregated statistics and their outlier masks - """ - class StatisticsCalculator(CalibrationCalculator): """ Component to calculate statistics from calibration events. - This class inherits from CalibrationCalculator and is responsible for + This class inherits from ``CalibrationCalculator`` and is responsible for calculating various statistics from calibration events, such as pedestal and flat-field data. It aggregates statistics, detects outliers, handles faulty data chunks. - The default option is to conduct only one pass over the data with non-overlapping - chunks, while overlapping chunks can be set by the ``chunk_shift`` parameter. - Two passes over the data, set via the ``second_pass``-flag, can be conducted - with a refined shift of the chunk in regions of trouble with a high percentage - of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. + The ``StatisticsCalculator`` holds two functions to conduct two different passes + over the data with and without overlapping chunks. The first pass is conducted + with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` + parameter in the second pass. The second pass over the data is only conducted + in regions of trouble with a high percentage of faulty pixels exceeding + the threshold ``faulty_pixels_threshold``. """ chunk_shift = Int( default_value=None, allow_none=True, help=( - "Number of samples to shift the aggregation chunk for the " - "calculation of the statistical values. If second_pass is set, " - "the first pass is conducted without overlapping chunks (chunk_shift=None) " - "and the second pass with a refined shift of the chunk in regions of trouble." - ), - ).tag(config=True) - - second_pass = Bool( - default_value=False, - help=( - "Set whether to conduct a second pass over the data " - "with a refined shift of the chunk in regions of trouble." + "Number of samples to shift the aggregation chunk for the calculation " + "of the statistical values. Only used in the second_pass(), since the " + "first_pass() is conducted without overlapping chunks (chunk_shift=None)." ), ).tag(config=True) @@ -185,42 +146,52 @@ class StatisticsCalculator(CalibrationCalculator): allow_none=True, help=( "Threshold in percentage of faulty pixels over the camera " - "to conduct second pass with a refined shift of the chunk " - "in regions of trouble." + "to identify regions of trouble." ), ).tag(config=True) - def __call__( + def first_pass( self, table, tel_id, masked_pixels_of_sample=None, col_name="image", ) -> Table: - # Check if the chunk_shift is set for second pass mode - if self.second_pass and self.chunk_shift is None: - raise ValueError( - "chunk_shift must be set if second pass over the data is selected" - ) + """ + Calculate the monitoring data for a given set of events with non-overlapping aggregation chunks. + + This method performs the first pass over the provided data table to calculate + various statistics for calibration purposes. The statistics are aggregated with + non-overlapping chunks (``chunk_shift`` set to None), and faulty pixels are detected + using a list of outlier detectors. + + + Parameters + ---------- + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and + timestamps of shape (n_images, ) + tel_id : int + Telescope ID for which the calibration is being performed + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing + col_name : str + Column name in the table from which the statistics will be aggregated + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics, their outlier masks, and the validity of the chunks + """ # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Pass through the whole provided dl1 table - if self.second_pass: - aggregated_stats = aggregator( - table=table, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=None, - ) - else: - aggregated_stats = aggregator( - table=table, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - + aggregated_stats = aggregator( + table=table, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=None, + ) # Detect faulty pixels with multiple instances of ``OutlierDetector`` outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) for aggregated_val, outlier_detector in self.outlier_detectors.items(): @@ -230,88 +201,143 @@ def __call__( ) # Add the outlier mask to the aggregated statistics aggregated_stats["outlier_mask"] = outlier_mask + # Get valid chunks and add them to the aggregated statistics + aggregated_stats["is_valid"] = self._get_valid_chunks(outlier_mask) + return aggregated_stats + + def second_pass( + self, + table, + valid_chunks, + tel_id, + masked_pixels_of_sample=None, + col_name="image", + ) -> Table: + """ + Conduct a second pass over the data to refine the statistics in regions with a high percentage of faulty pixels. + + This method performs a second pass over the data with a refined shift of the chunk in regions where a high percentage + of faulty pixels were detected during the first pass. Note: Multiple first passes of different calibration events are + performed which may lead to different identification of faulty chunks in rare cases. Therefore a joined list of faulty + chunks is recommended to be passed to the second pass(es) if those different passes use the same ``chunk_size``. + Parameters + ---------- + table : astropy.table.Table + DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and timestamps of shape (n_images, ). + valid_chunks : ndarray + Boolean array indicating the validity of each chunk from the first pass. + Note: This boolean array can be a ``logical_and`` from multiple first passes of different calibration events. + tel_id : int + Telescope ID for which the calibration is being performed. + masked_pixels_of_sample : ndarray, optional + Boolean array of masked pixels of shape (n_pix, ) that are not available for processing. + col_name : str + Column name in the table from which the statistics will be aggregated. + + Returns + ------- + astropy.table.Table + Table containing the aggregated statistics after the second pass, their outlier masks, and the validity of the chunks. + """ + # Check if the chunk_shift is set for the second pass + if self.chunk_shift is None: + raise ValueError( + "chunk_shift must be set if second pass over the data is requested" + ) + # Get the aggregator + aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data - if self.second_pass: - # Check if the camera has two gain channels - if outlier_mask.shape[1] == 2: - # Combine the outlier mask of both gain channels - outlier_mask = np.logical_or( - outlier_mask[:, 0, :], - outlier_mask[:, 1, :], + aggregated_stats_secondpass = None + if np.all(valid_chunks): + self.log.info( + "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + ) + else: + chunk_size = aggregator.chunk_size + faulty_chunks_indices = np.where(~valid_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunk detected in the first pass at index '{index}'." ) - # Calculate the fraction of faulty pixels over the camera - faulty_pixels_percentage = ( - np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] - ) * 100.0 - - # Check for faulty chunks if the threshold is exceeded - faulty_chunks = faulty_pixels_percentage > self.faulty_pixels_threshold - if np.any(faulty_chunks): - chunk_size = aggregated_stats["n_events"][0] - faulty_chunks_indices = np.where(faulty_chunks)[0] - for index in faulty_chunks_indices: - # Log information of the faulty chunks - self.log.warning( - f"Faulty chunk ({int(faulty_pixels_percentage[index])}% of the camera unavailable) detected in the first pass: time_start={aggregated_stats['time_start'][index]}; time_end={aggregated_stats['time_end'][index]}" - ) - # Calculate the start of the slice depending on whether the previous chunk was faulty or not - slice_start = ( - chunk_size * index - if index - 1 in faulty_chunks_indices - else chunk_size * (index - 1) + # Calculate the start of the slice depending on whether the previous chunk was faulty or not + slice_start = ( + chunk_size * index + if index - 1 in faulty_chunks_indices + else chunk_size * (index - 1) + ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) + # Slice the dl1 table according to the previously calculated start and end. + table_sliced = table[slice_start:slice_end] + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + # Checking for the length of the sliced table to be greater than he chunk_size + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) - # Set the start of the slice to the first element of the dl1 table if out of bound - # and add one ``chunk_shift``. - slice_start = max(0, slice_start) + self.chunk_shift - # Set the end of the slice to the last element of the dl1 table if out of bound - # and subtract one ``chunk_shift``. - slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( - self.chunk_shift - 1 + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) - # Slice the dl1 table according to the previously calculated start and end. - table_sliced = table[slice_start:slice_end] + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) + return aggregated_stats_secondpass + + def _get_valid_chunks(self, outlier_mask): + """ + Identify valid chunks based on the outlier mask. - # Run the stats aggregator on the sliced dl1 table with a chunk_shift - # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than he chunk_size - # since it can be smaller if the last two chunks are faulty. - if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) + This method processes the outlier mask to determine which chunks of data + are considered valid or faulty. A chunk is marked as faulty if the percentage + of outlier pixels exceeds a predefined threshold ``faulty_pixels_threshold``. - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool - ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector( - aggregated_stats_secondpass[aggregated_val] - ), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass[ - "outlier_mask" - ] = outlier_mask_secondpass + Parameters + ---------- + outlier_mask : numpy.ndarray + Boolean array indicating outlier pixels. The shape of the array should + match the shape of the aggregated statistics. - # Stack the aggregated statistics of the second pass to the first pass - aggregated_stats = vstack( - [aggregated_stats, aggregated_stats_secondpass] - ) - # Sort the aggregated statistics based on the starting time - aggregated_stats.sort(["time_start"]) - else: - self.log.info( - "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." - ) - # Return the aggregated statistics and their outlier masks - return aggregated_stats + Returns + ------- + numpy.ndarray + Boolean array where each element indicates whether the corresponding + chunk is valid (True) or faulty (False). + """ + # Check if the camera has two gain channels + if outlier_mask.shape[1] == 2: + # Combine the outlier mask of both gain channels + outlier_mask = np.logical_or( + outlier_mask[:, 0, :], + outlier_mask[:, 1, :], + ) + # Calculate the fraction of faulty pixels over the camera + faulty_pixels_percentage = ( + np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] + ) * 100.0 + # Check for valid chunks if the threshold is not exceeded + valid_chunks = faulty_pixels_percentage < self.faulty_pixels_threshold + return valid_chunks diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 08c818de0af..37e8fb8fff6 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -3,7 +3,7 @@ """ import numpy as np -from astropy.table import Table +from astropy.table import Table, vstack from astropy.time import Time from traitlets.config.loader import Config @@ -11,8 +11,8 @@ from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator -def test_onepass_calculator(example_subarray): - """test basic 'one pass' functionality of the StatisticsCalculator""" +def test_statistics_calculator(example_subarray): + """test basic functionality of the StatisticsCalculator""" # Create dummy data for testing times = Time( @@ -26,31 +26,40 @@ def test_onepass_calculator(example_subarray): [times, event_ids, charge_data], names=("time_mono", "event_id", "image"), ) - # Initialize the aggregators and calculators - chunk_size = 500 - aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + # Initialize the aggregator and calculator + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) calculator = CalibrationCalculator.from_name( name="StatisticsCalculator", subarray=example_subarray, stats_aggregator=aggregator, - ) - calculator_chunk_shift = StatisticsCalculator( - subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=250 + chunk_shift=100, ) # Compute the statistical values - stats = calculator(table=charge_table, tel_id=1) - stats_chunk_shift = calculator_chunk_shift(table=charge_table, tel_id=1) - + stats = calculator.first_pass(table=charge_table, tel_id=1) + # Set all chunks as faulty to aggregate the statistic values with a "global" chunk shift + valid_chunks = np.zeros_like(stats["is_valid"].data, dtype=bool) + # Run the second pass over the data + stats_chunk_shift = calculator.second_pass( + table=charge_table, valid_chunks=valid_chunks, tel_id=1 + ) + # Stack the statistic values from the first and second pass + stats_combined = vstack([stats, stats_chunk_shift]) + # Sort the combined aggregated statistic values by starting time + stats_combined.sort(["time_start"]) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) np.testing.assert_allclose(stats[1]["median"], 77.0, atol=2.5) np.testing.assert_allclose(stats[0]["std"], 10.0, atol=2.5) - # Check if three chunks are used for the computation of aggregated statistic values as the last chunk overflows - assert len(stats) * 2 == len(stats_chunk_shift) + 1 + np.testing.assert_allclose(stats_chunk_shift[0]["mean"], 77.0, atol=2.5) + np.testing.assert_allclose(stats_chunk_shift[1]["median"], 77.0, atol=2.5) + np.testing.assert_allclose(stats_chunk_shift[0]["std"], 10.0, atol=2.5) + # Check if overlapping chunks of the second pass were aggregated + assert stats_chunk_shift is not None + assert len(stats_combined) > len(stats) -def test_secondpass_calculator(example_subarray): +def test_outlier_detector(example_subarray): """test the chunk shift option and the boundary case for the last chunk""" # Create dummy data for testing @@ -89,18 +98,30 @@ def test_secondpass_calculator(example_subarray): "validity_range": [2.0, 8.0], }, ], - "chunk_shift": 100, - "second_pass": True, - "faulty_pixels_threshold": 1.0, + "chunk_shift": 500, + "faulty_pixels_threshold": 9.0, }, "SigmaClippingAggregator": { - "chunk_size": 500, + "chunk_size": 1000, }, } ) # Initialize the calculator from config calculator = StatisticsCalculator(subarray=example_subarray, config=config) - # Compute aggregated statistic values - stats = calculator(ped_table, 1, col_name="image") - # Check if the second pass was activated - assert len(stats) > 20 + # Run the first pass over the data + stats_first_pass = calculator.first_pass(table=ped_table, tel_id=1) + # Run the second pass over the data + stats_second_pass = calculator.second_pass( + table=ped_table, valid_chunks=stats_first_pass["is_valid"].data, tel_id=1 + ) + stats_combined = vstack([stats_first_pass, stats_second_pass]) + # Sort the combined aggregated statistic values by starting time + stats_combined.sort(["time_start"]) + # Check if overlapping chunks of the second pass were aggregated + assert stats_second_pass is not None + assert len(stats_combined) > len(stats_second_pass) + # Check if the calculated statistical values are reasonable + # for a camera with two gain channels + np.testing.assert_allclose(stats_combined[0]["mean"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_combined[1]["median"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_combined[0]["std"], 5.0, atol=2.5) From 88e21266c5ce5a9956db65ee3aff03e5a5c680d1 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 26 Aug 2024 15:51:20 +0200 Subject: [PATCH 193/221] fix docs and add changelog --- docs/api-reference/monitoring/calculator.rst | 11 +++++++++++ docs/api-reference/monitoring/index.rst | 4 ++-- docs/changes/2609.features.rst | 1 + src/ctapipe/monitoring/calculator.py | 4 ++-- 4 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 docs/api-reference/monitoring/calculator.rst create mode 100644 docs/changes/2609.features.rst diff --git a/docs/api-reference/monitoring/calculator.rst b/docs/api-reference/monitoring/calculator.rst new file mode 100644 index 00000000000..93a1c1ec861 --- /dev/null +++ b/docs/api-reference/monitoring/calculator.rst @@ -0,0 +1,11 @@ +.. _calibration_calculator: + +********************** +Calibration Calculator +********************** + + +Reference/API +============= + +.. automodapi:: ctapipe.monitoring.calculator diff --git a/docs/api-reference/monitoring/index.rst b/docs/api-reference/monitoring/index.rst index c38bf7bb3a9..032c26d8fff 100644 --- a/docs/api-reference/monitoring/index.rst +++ b/docs/api-reference/monitoring/index.rst @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment. -Currently, only code related to :ref:`stats_aggregator` and :ref:`outlier_detector` is implemented here. +Currently, code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. Submodules @@ -20,7 +20,7 @@ Submodules :maxdepth: 1 aggregator - interpolation + calculator outlier diff --git a/docs/changes/2609.features.rst b/docs/changes/2609.features.rst new file mode 100644 index 00000000000..fac55b285f6 --- /dev/null +++ b/docs/changes/2609.features.rst @@ -0,0 +1 @@ +Add calibration calculators which aggregates statistics, detects outliers, handles faulty data chunks. diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 9359e0b1689..89cb5b2c3ed 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -126,7 +126,7 @@ class StatisticsCalculator(CalibrationCalculator): The ``StatisticsCalculator`` holds two functions to conduct two different passes over the data with and without overlapping chunks. The first pass is conducted with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` - parameter in the second pass. The second pass over the data is only conducted + parameter for the second pass. The second pass over the data is only conducted in regions of trouble with a high percentage of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ @@ -279,7 +279,7 @@ def second_pass( table_sliced = table[slice_start:slice_end] # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than he chunk_size + # Checking for the length of the sliced table to be greater than the chunk_size # since it can be smaller if the last two chunks are faulty. if len(table_sliced) > aggregator.chunk_size: aggregated_stats_secondpass = aggregator( From 5b8a0b3bbd343e4db8a52afc62985b4fe442bcd6 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 27 Aug 2024 10:58:45 +0200 Subject: [PATCH 194/221] removed check for any faulty chunk check should be done before and then the second pass should not be called if only valid chunks are provided --- src/ctapipe/monitoring/calculator.py | 100 +++++++++++++-------------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 89cb5b2c3ed..20da80d7428 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -249,62 +249,56 @@ def second_pass( aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data aggregated_stats_secondpass = None - if np.all(valid_chunks): - self.log.info( - "No faulty chunks detected in the first pass. The second pass with a finer chunk shift is not executed." + faulty_chunks_indices = np.where(~valid_chunks)[0] + for index in faulty_chunks_indices: + # Log information of the faulty chunks + self.log.warning( + f"Faulty chunk detected in the first pass at index '{index}'." ) - else: - chunk_size = aggregator.chunk_size - faulty_chunks_indices = np.where(~valid_chunks)[0] - for index in faulty_chunks_indices: - # Log information of the faulty chunks - self.log.warning( - f"Faulty chunk detected in the first pass at index '{index}'." - ) - # Calculate the start of the slice depending on whether the previous chunk was faulty or not - slice_start = ( - chunk_size * index - if index - 1 in faulty_chunks_indices - else chunk_size * (index - 1) - ) - # Set the start of the slice to the first element of the dl1 table if out of bound - # and add one ``chunk_shift``. - slice_start = max(0, slice_start) + self.chunk_shift - # Set the end of the slice to the last element of the dl1 table if out of bound - # and subtract one ``chunk_shift``. - slice_end = min(len(table) - 1, chunk_size * (index + 2)) - ( - self.chunk_shift - 1 - ) - # Slice the dl1 table according to the previously calculated start and end. - table_sliced = table[slice_start:slice_end] - # Run the stats aggregator on the sliced dl1 table with a chunk_shift - # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than the chunk_size - # since it can be smaller if the last two chunks are faulty. - if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool + # Calculate the start of the slice depending on whether the previous chunk was faulty or not + slice_start = ( + aggregator.chunk_size * index + if index - 1 in faulty_chunks_indices + else aggregator.chunk_size * (index - 1) + ) + # Set the start of the slice to the first element of the dl1 table if out of bound + # and add one ``chunk_shift``. + slice_start = max(0, slice_start) + self.chunk_shift + # Set the end of the slice to the last element of the dl1 table if out of bound + # and subtract one ``chunk_shift``. + slice_end = min(len(table) - 1, aggregator.chunk_size * (index + 2)) - ( + self.chunk_shift - 1 + ) + # Slice the dl1 table according to the previously calculated start and end. + table_sliced = table[slice_start:slice_end] + # Run the stats aggregator on the sliced dl1 table with a chunk_shift + # to sample the period of trouble (carflashes etc.) as effectively as possible. + # Checking for the length of the sliced table to be greater than the ``chunk_size`` + # since it can be smaller if the last two chunks are faulty. + if len(table_sliced) > aggregator.chunk_size: + aggregated_stats_secondpass = aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass - aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( - outlier_mask_secondpass + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) return aggregated_stats_secondpass def _get_valid_chunks(self, outlier_mask): From d6681fc288970bced1c49fd36d81fd3806bfb277 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 27 Aug 2024 18:08:17 +0200 Subject: [PATCH 195/221] removed base class CalibrationCalculator and only have one StatisticsCalculator class --- docs/api-reference/monitoring/index.rst | 2 +- src/ctapipe/monitoring/aggregator.py | 2 +- src/ctapipe/monitoring/calculator.py | 86 +++++++------------ .../monitoring/tests/test_calculator.py | 5 +- 4 files changed, 37 insertions(+), 58 deletions(-) diff --git a/docs/api-reference/monitoring/index.rst b/docs/api-reference/monitoring/index.rst index 032c26d8fff..11531d94597 100644 --- a/docs/api-reference/monitoring/index.rst +++ b/docs/api-reference/monitoring/index.rst @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment. -Currently, code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. +Code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. Submodules diff --git a/src/ctapipe/monitoring/aggregator.py b/src/ctapipe/monitoring/aggregator.py index 9dc3aa485fd..8102b21e855 100644 --- a/src/ctapipe/monitoring/aggregator.py +++ b/src/ctapipe/monitoring/aggregator.py @@ -57,7 +57,7 @@ def __call__( and call the relevant function of the particular aggregator to compute aggregated statistic values. The chunks are generated in a way that ensures they do not overflow the bounds of the table. - If ``chunk_shift`` is None, chunks will not overlap, but the last chunk is ensured to be - of size `chunk_size`, even if it means the last two chunks will overlap. + of size ``chunk_size``, even if it means the last two chunks will overlap. - If ``chunk_shift`` is provided, it will determine the number of samples to shift between the start of consecutive chunks resulting in an overlap of chunks. Chunks that overflows the bounds of the table are not considered. diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index 20da80d7428..a3adc01d370 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -1,5 +1,5 @@ """ -Definition of the ``CalibrationCalculator`` classes, providing all steps needed to +Definition of the ``StatisticsCalculator`` class, providing all steps needed to calculate the montoring data for the camera calibration. """ @@ -19,26 +19,22 @@ from ctapipe.monitoring.outlier import OutlierDetector __all__ = [ - "CalibrationCalculator", "StatisticsCalculator", ] -class CalibrationCalculator(TelescopeComponent): +class StatisticsCalculator(TelescopeComponent): """ - Base component for calibration calculators. - - This class provides the foundational methods and attributes for - calculating camera-related monitoring data. It is designed - to be extended by specific calibration calculators that implement - the required methods for different types of calibration. + Component to calculate statistics from calibration events. - Attributes - ---------- - stats_aggregator_type : ctapipe.core.traits.TelescopeParameter - The type of StatisticsAggregator to be used for aggregating statistics. - outlier_detector_list : list of dict - List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector. + The ``StatisticsCalculator`` is responsible for calculating various statistics from + calibration events, such as pedestal and flat-field data. It aggregates statistics, + detects outliers, and handles faulty data periods. + This class holds two functions to conduct two different passes over the data with and without + overlapping aggregation chunks. The first pass is conducted with non-overlapping chunks, + while overlapping chunks can be set by the ``chunk_shift`` parameter for the second pass. + The second pass over the data is only conducted in regions of trouble with a high percentage + of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. """ stats_aggregator_type = TelescopeParameter( @@ -56,7 +52,27 @@ class CalibrationCalculator(TelescopeComponent): help=( "List of dicts containing the name of the OutlierDetector subclass to be used, " "the aggregated statistic value to which the detector should be applied, " - "and the validity range of the detector." + "and the validity range of the detector. " + "E.g. ``[{'apply_to': 'std', 'name': 'RangeOutlierDetector', 'validity_range': [2.0, 8.0]},]``." + ), + ).tag(config=True) + + chunk_shift = Int( + default_value=None, + allow_none=True, + help=( + "Number of samples to shift the aggregation chunk for the calculation " + "of the statistical values. Only used in the second_pass(), since the " + "first_pass() is conducted with non-overlapping chunks (chunk_shift=None)." + ), + ).tag(config=True) + + faulty_pixels_threshold = Float( + default_value=10.0, + allow_none=True, + help=( + "Threshold in percentage of faulty pixels over the camera " + "to identify regions of trouble." ), ).tag(config=True) @@ -114,42 +130,6 @@ def __init__( parent=self, ) - -class StatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - - This class inherits from ``CalibrationCalculator`` and is responsible for - calculating various statistics from calibration events, such as pedestal - and flat-field data. It aggregates statistics, detects outliers, - handles faulty data chunks. - The ``StatisticsCalculator`` holds two functions to conduct two different passes - over the data with and without overlapping chunks. The first pass is conducted - with non-overlapping, while overlapping chunks can be set by the ``chunk_shift`` - parameter for the second pass. The second pass over the data is only conducted - in regions of trouble with a high percentage of faulty pixels exceeding - the threshold ``faulty_pixels_threshold``. - """ - - chunk_shift = Int( - default_value=None, - allow_none=True, - help=( - "Number of samples to shift the aggregation chunk for the calculation " - "of the statistical values. Only used in the second_pass(), since the " - "first_pass() is conducted without overlapping chunks (chunk_shift=None)." - ), - ).tag(config=True) - - faulty_pixels_threshold = Float( - default_value=10.0, - allow_none=True, - help=( - "Threshold in percentage of faulty pixels over the camera " - "to identify regions of trouble." - ), - ).tag(config=True) - def first_pass( self, table, @@ -252,7 +232,7 @@ def second_pass( faulty_chunks_indices = np.where(~valid_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks - self.log.warning( + self.log.info( f"Faulty chunk detected in the first pass at index '{index}'." ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 37e8fb8fff6..876bfc3955c 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -8,7 +8,7 @@ from traitlets.config.loader import Config from ctapipe.monitoring.aggregator import PlainAggregator -from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator +from ctapipe.monitoring.calculator import StatisticsCalculator def test_statistics_calculator(example_subarray): @@ -28,8 +28,7 @@ def test_statistics_calculator(example_subarray): ) # Initialize the aggregator and calculator aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) - calculator = CalibrationCalculator.from_name( - name="StatisticsCalculator", + calculator = StatisticsCalculator( subarray=example_subarray, stats_aggregator=aggregator, chunk_shift=100, From 54ab5e94ab5cecd9f7f7830696d1700ab68a450a Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 09:17:49 +0200 Subject: [PATCH 196/221] fix fstring in logging --- src/ctapipe/monitoring/calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index a3adc01d370..b58c4259b70 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -233,7 +233,7 @@ def second_pass( for index in faulty_chunks_indices: # Log information of the faulty chunks self.log.info( - f"Faulty chunk detected in the first pass at index '{index}'." + "Faulty chunk detected in the first pass at index '%s'.", index ) # Calculate the start of the slice depending on whether the previous chunk was faulty or not slice_start = ( From 0edff6478fe95016d4a0a9460e174ab54467583d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 17:52:56 +0200 Subject: [PATCH 197/221] bug fix aggregated stats of the second pass we simply overwritten rather than append and vstacked() unit tests are improved to properly test for the correct number of chunks --- src/ctapipe/monitoring/calculator.py | 58 +++++++++++-------- .../monitoring/tests/test_calculator.py | 53 ++++++++++++----- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py index b58c4259b70..d95041cf75c 100644 --- a/src/ctapipe/monitoring/calculator.py +++ b/src/ctapipe/monitoring/calculator.py @@ -4,7 +4,7 @@ """ import numpy as np -from astropy.table import Table +from astropy.table import Table, vstack from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( @@ -225,10 +225,15 @@ def second_pass( raise ValueError( "chunk_shift must be set if second pass over the data is requested" ) + # Check if at least one chunk is faulty + if np.all(valid_chunks): + raise ValueError( + "All chunks are valid. The second pass over the data is redundant." + ) # Get the aggregator aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] # Conduct a second pass over the data - aggregated_stats_secondpass = None + aggregated_stats_secondpass = [] faulty_chunks_indices = np.where(~valid_chunks)[0] for index in faulty_chunks_indices: # Log information of the faulty chunks @@ -254,31 +259,36 @@ def second_pass( # Run the stats aggregator on the sliced dl1 table with a chunk_shift # to sample the period of trouble (carflashes etc.) as effectively as possible. # Checking for the length of the sliced table to be greater than the ``chunk_size`` - # since it can be smaller if the last two chunks are faulty. + # since it can be smaller if the last two chunks are faulty. Note: The two last chunks + # can be overlapping during the first pass, so we simply ignore them if there are faulty. if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass = aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool - ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), + aggregated_stats_secondpass.append( + aggregator( + table=table_sliced, + masked_pixels_of_sample=masked_pixels_of_sample, + col_name=col_name, + chunk_shift=self.chunk_shift, + ) ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass - aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( - outlier_mask_secondpass + # Stack the aggregated statistics of each faulty chunk + aggregated_stats_secondpass = vstack(aggregated_stats_secondpass) + # Detect faulty pixels with multiple instances of OutlierDetector of the second pass + outlier_mask_secondpass = np.zeros_like( + aggregated_stats_secondpass["mean"], dtype=bool + ) + for ( + aggregated_val, + outlier_detector, + ) in self.outlier_detectors.items(): + outlier_mask_secondpass = np.logical_or( + outlier_mask_secondpass, + outlier_detector(aggregated_stats_secondpass[aggregated_val]), ) + # Add the outlier mask to the aggregated statistics + aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass + aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( + outlier_mask_secondpass + ) return aggregated_stats_secondpass def _get_valid_chunks(self, outlier_mask): diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 876bfc3955c..520254724ed 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -15,23 +15,26 @@ def test_statistics_calculator(example_subarray): """test basic functionality of the StatisticsCalculator""" # Create dummy data for testing + n_images = 5050 times = Time( - np.linspace(60117.911, 60117.9258, num=5000), scale="tai", format="mjd" + np.linspace(60117.911, 60117.9258, num=n_images), scale="tai", format="mjd" ) - event_ids = np.linspace(35, 725000, num=5000, dtype=int) + event_ids = np.linspace(35, 725000, num=n_images, dtype=int) rng = np.random.default_rng(0) - charge_data = rng.normal(77.0, 10.0, size=(5000, 2, 1855)) + charge_data = rng.normal(77.0, 10.0, size=(n_images, 2, 1855)) # Create tables charge_table = Table( [times, event_ids, charge_data], names=("time_mono", "event_id", "image"), ) # Initialize the aggregator and calculator - aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000) + chunk_size = 1000 + aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) + chunk_shift = 500 calculator = StatisticsCalculator( subarray=example_subarray, stats_aggregator=aggregator, - chunk_shift=100, + chunk_shift=chunk_shift, ) # Compute the statistical values stats = calculator.first_pass(table=charge_table, tel_id=1) @@ -42,9 +45,12 @@ def test_statistics_calculator(example_subarray): table=charge_table, valid_chunks=valid_chunks, tel_id=1 ) # Stack the statistic values from the first and second pass - stats_combined = vstack([stats, stats_chunk_shift]) - # Sort the combined aggregated statistic values by starting time - stats_combined.sort(["time_start"]) + stats_stacked = vstack([stats, stats_chunk_shift]) + # Sort the stacked aggregated statistic values by starting time + stats_stacked.sort(["time_start"]) + print(stats) + print(stats_chunk_shift) + print(stats_stacked) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) @@ -55,7 +61,21 @@ def test_statistics_calculator(example_subarray): np.testing.assert_allclose(stats_chunk_shift[0]["std"], 10.0, atol=2.5) # Check if overlapping chunks of the second pass were aggregated assert stats_chunk_shift is not None - assert len(stats_combined) > len(stats) + # Check if the number of aggregated chunks is correct + # In the first pass, the number of chunks is equal to the + # number of images divided by the chunk size plus one + # overlapping chunk at the end. + expected_len_firstpass = n_images // chunk_size + 1 + assert len(stats) == expected_len_firstpass + # In the second pass, the number of chunks is equal to the + # number of images divided by the chunk shift minus the + # number of chunks in the first pass, since we set all + # chunks to be faulty. + expected_len_secondpass = (n_images // chunk_shift) - expected_len_firstpass + assert len(stats_chunk_shift) == expected_len_secondpass + # The total number of aggregated chunks is the sum of the + # number of chunks in the first and second pass. + assert len(stats_stacked) == expected_len_firstpass + expected_len_secondpass def test_outlier_detector(example_subarray): @@ -113,14 +133,15 @@ def test_outlier_detector(example_subarray): stats_second_pass = calculator.second_pass( table=ped_table, valid_chunks=stats_first_pass["is_valid"].data, tel_id=1 ) - stats_combined = vstack([stats_first_pass, stats_second_pass]) - # Sort the combined aggregated statistic values by starting time - stats_combined.sort(["time_start"]) + # Stack the statistic values from the first and second pass + stats_stacked = vstack([stats_first_pass, stats_second_pass]) + # Sort the stacked aggregated statistic values by starting time + stats_stacked.sort(["time_start"]) # Check if overlapping chunks of the second pass were aggregated assert stats_second_pass is not None - assert len(stats_combined) > len(stats_second_pass) + assert len(stats_stacked) > len(stats_second_pass) # Check if the calculated statistical values are reasonable # for a camera with two gain channels - np.testing.assert_allclose(stats_combined[0]["mean"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_combined[1]["median"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_combined[0]["std"], 5.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[0]["mean"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[1]["median"], 2.0, atol=2.5) + np.testing.assert_allclose(stats_stacked[0]["std"], 5.0, atol=2.5) From c41b0b71ea5bbdb7df5ca6dcf1e7ef5044b227d0 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 28 Aug 2024 18:00:19 +0200 Subject: [PATCH 198/221] removed print --- src/ctapipe/monitoring/tests/test_calculator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py index 520254724ed..b64630f2597 100644 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ b/src/ctapipe/monitoring/tests/test_calculator.py @@ -48,9 +48,6 @@ def test_statistics_calculator(example_subarray): stats_stacked = vstack([stats, stats_chunk_shift]) # Sort the stacked aggregated statistic values by starting time stats_stacked.sort(["time_start"]) - print(stats) - print(stats_chunk_shift) - print(stats_stacked) # Check if the calculated statistical values are reasonable # for a camera with two gain channels np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) From 4d8d89679bb008039281d5c730a5453ae873b806 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 3 Sep 2024 12:14:05 +0200 Subject: [PATCH 199/221] removed me from CODEOWNERS --- CODEOWNERS | 2 -- 1 file changed, 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 70c53117a07..06c29bc6c78 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,8 +5,6 @@ ctapipe/calib/camera @watsonjj ctapipe/image/extractor.py @watsonjj @HealthyPear -ctapipe/monitoring @TjarkMiener - ctapipe/reco/HillasReconstructor.py @HealthyPear ctapipe/reco/tests/test_HillasReconstructor.py @HealthyPear From fe1dfb92fca4cccbf3559cc1f5213388932103b1 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:31:20 +0200 Subject: [PATCH 200/221] Copying over code for interpolators and pointing calculators --- src/ctapipe/calib/camera/calibrator.py | 206 ++++++++++++- src/ctapipe/image/psf_model.py | 95 ++++++ src/ctapipe/io/interpolation.py | 345 ++++++++++++++++++++++ src/ctapipe/io/tests/test_interpolator.py | 179 +++++++++++ 4 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 src/ctapipe/image/psf_model.py create mode 100644 src/ctapipe/io/interpolation.py create mode 100644 src/ctapipe/io/tests/test_interpolator.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 853ba3f7da8..8d9e309c077 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -7,20 +7,34 @@ import astropy.units as u import numpy as np +import Vizier +from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Int, + Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = ["CameraCalibrator"] +__all__ = [ + "CalibrationCalculator", + "TwoPassStatisticsCalculator", + "CameraCalibrator", +] @cache @@ -48,6 +62,196 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + output_path = Path(help="output filename").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_extractor=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class TwoPassStatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) + + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) + + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > self.faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously calculated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + def _get_slice_range( + self, + chunk_nr, + chunk_size, + faultless_previous_chunk, + last_chunk, + last_element, + ): + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + self.chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + self.chunk_shift + ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + + return slice_start, slice_stop + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py new file mode 100644 index 00000000000..458070b8145 --- /dev/null +++ b/src/ctapipe/image/psf_model.py @@ -0,0 +1,95 @@ +""" +Models for the Point Spread Functions of the different telescopes +""" + +__all__ = ["PSFModel", "ComaModel"] + +from abc import abstractmethod + +import numpy as np +from scipy.stats import laplace, laplace_asymmetric +from traitlets import List + + +class PSFModel: + def __init__(self, **kwargs): + """ + Base component to describe image distortion due to the optics of the different cameras. + """ + + @classmethod + def from_name(cls, name, **kwargs): + """ + Obtain an instance of a subclass via its name + + Parameters + ---------- + name : str + Name of the subclass to obtain + + Returns + ------- + Instance + Instance of subclass to this class + """ + requested_subclass = cls.non_abstract_subclasses()[name] + return requested_subclass(**kwargs) + + @abstractmethod + def pdf(self, *args): + pass + + @abstractmethod + def update_model_parameters(self, *args): + pass + + +class ComaModel(PSFModel): + """ + PSF model, describing pure coma aberrations PSF effect + """ + + asymmetry_params = List( + default_value=[0.49244797, 9.23573115, 0.15216096], + help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", + ).tag(config=True) + radial_scale_params = List( + default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", + ).tag(config=True) + az_scale_params = List( + default_value=[0.24271557, 7.5511501, 0.02037972], + help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", + ).tag(config=True) + + def k_func(self, x): + return ( + 1 + - self.asymmetry_params[0] * np.tanh(self.asymmetry_params[1] * x) + - self.asymmetry_params[2] * x + ) + + def sr_func(self, x): + return ( + self.radial_scale_params[0] + - self.radial_scale_params[1] * x + + self.radial_scale_params[2] * x**2 + - self.radial_scale_params[3] * x**3 + ) + + def sf_func(self, x): + return self.az_scale_params[0] * np.exp( + -self.az_scale_params[1] * x + ) + self.az_scale_params[2] / (self.az_scale_params[2] + x) + + def pdf(self, r, f): + return laplace_asymmetric.pdf(r, *self.radial_pdf_params) * laplace.pdf( + f, *self.azimuthal_pdf_params + ) + + def update_model_parameters(self, r, f): + k = self.k_func(r) + sr = self.sr_func(r) + sf = self.sf_func(r) + self.radial_pdf_params = (k, r, sr) + self.azimuthal_pdf_params = (f, sf) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py new file mode 100644 index 00000000000..3b792c2107d --- /dev/null +++ b/src/ctapipe/io/interpolation.py @@ -0,0 +1,345 @@ +from abc import ABCMeta, abstractmethod +from typing import Any + +import astropy.units as u +import numpy as np +import tables +from astropy.time import Time +from scipy.interpolate import interp1d + +from ctapipe.core import Component, traits + +from .astropy_helpers import read_table + + +class StepFunction: + + """ + Step function Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the closest previous + point given. + + Parameters + ---------- + values : None | np.array + Numpy array of the data that is to be interpolated. + The first dimension needs to be an index over time + times : None | np.array + Time values over which data are to be interpolated + need to be sorted and have same length as first dimension of values + """ + + def __init__( + self, + times, + values, + bounds_error=True, + fill_value="extrapolate", + assume_sorted=True, + copy=False, + ): + self.values = values + self.times = times + self.bounds_error = bounds_error + self.fill_value = fill_value + + def __call__(self, point): + if point < self.times[0]: + if self.bounds_error: + raise ValueError("below the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[0] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + else: + i = np.searchsorted(self.times, point, side="left") + return self.values[i - 1] + + +class Interpolator(Component, metaclass=ABCMeta): + """ + Interpolator parent class. + + Parameters + ---------- + h5file : None | tables.File + A open hdf5 file with read access. + """ + + bounds_error = traits.Bool( + default_value=True, + help="If true, raises an exception when trying to extrapolate out of the given table", + ).tag(config=True) + + extrapolate = traits.Bool( + help="If bounds_error is False, this flag will specify whether values outside" + "the available values are filled with nan (False) or extrapolated (True).", + default_value=False, + ).tag(config=True) + + telescope_data_group = None + required_columns = set() + expected_units = {} + + def __init__(self, h5file=None, **kwargs): + super().__init__(**kwargs) + + if h5file is not None and not isinstance(h5file, tables.File): + raise TypeError("h5file must be a tables.File") + self.h5file = h5file + + self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) + if self.bounds_error: + self.interp_options["bounds_error"] = True + elif self.extrapolate: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = "extrapolate" + else: + self.interp_options["bounds_error"] = False + self.interp_options["fill_value"] = np.nan + + self._interpolators = {} + + @abstractmethod + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + This method reads input tables and creates instances of the needed interpolators + to be added to _interpolators. The first index of _interpolators needs to be + tel_id, the second needs to be the name of the parameter that is to be interpolated + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are always ``time`` as ``Time`` column and + other columns for the data that is to be interpolated + """ + + pass + + def _check_tables(self, input_table): + missing = self.required_columns - set(input_table.colnames) + if len(missing) > 0: + raise ValueError(f"Table is missing required column(s): {missing}") + for col in self.expected_units: + unit = input_table[col].unit + if unit is None: + if self.expected_units[col] is not None: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + elif not self.expected_units[col].is_equivalent(unit): + if self.expected_units[col] is None: + raise ValueError(f"{col} must have units compatible with 'None'") + else: + raise ValueError( + f"{col} must have units compatible with '{self.expected_units[col].name}'" + ) + + def _check_interpolators(self, tel_id): + if tel_id not in self._interpolators: + if self.h5file is not None: + self._read_parameter_table(tel_id) # might need to be removed + else: + raise KeyError(f"No table available for tel_id {tel_id}") + + def _read_parameter_table(self, tel_id): + input_table = read_table( + self.h5file, + f"{self.telescope_data_group}/tel_{tel_id:03d}", + ) + self.add_table(tel_id, input_table) + + +class PointingInterpolator(Interpolator): + """ + Interpolator for pointing and pointing correction data + """ + + telescope_data_group = "/dl0/monitoring/telescope/pointing" + required_columns = frozenset(["time", "azimuth", "altitude"]) + expected_units = {"azimuth": u.rad, "altitude": u.rad} + + def __call__(self, tel_id, time): + """ + Interpolate alt/az for given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the pointing + + Returns + ------- + altitude : astropy.units.Quantity[deg] + interpolated altitude angle + azimuth : astropy.units.Quantity[deg] + interpolated azimuth angle + """ + + self._check_interpolators(tel_id) + + mjd = time.tai.mjd + az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) + alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) + return alt, az + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` + as quantity columns for pointing and pointing correction data. + """ + + self._check_tables(input_table) + + if not isinstance(input_table["time"], Time): + raise TypeError("'time' column of pointing table must be astropy.time.Time") + + input_table = input_table.copy() + input_table.sort("time") + + az = input_table["azimuth"].quantity.to_value(u.rad) + # prepare azimuth for interpolation by "unwrapping": i.e. turning + # [359, 1] into [359, 361]. This assumes that if we get values like + # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees + # the other way around. This should be true for all telescopes given + # the sampling speed of pointing values and their maximum movement speed. + # No telescope can turn more than 180° in 2 seconds. + az = np.unwrap(az) + alt = input_table["altitude"].quantity.to_value(u.rad) + mjd = input_table["time"].tai.mjd + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) + self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) + + +class FlatFieldInterpolator(Interpolator): + """ + Interpolator for flatfield data + """ + + telescope_data_group = "dl1/calibration/gain" # TBD + required_columns = frozenset(["time", "gain"]) # TBD + expected_units = {"gain": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate flatfield data for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + ffield : array [float] + interpolated flatfield data + """ + + self._check_interpolators(tel_id) + + ffield = self._interpolators[tel_id]["gain"](time) + return ffield + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "gain" + for the flatfield data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + gain = input_table["gain"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["gain"] = StepFunction( + time, gain, **self.interp_options + ) + + +class PedestalInterpolator(Interpolator): + """ + Interpolator for Pedestal data + """ + + telescope_data_group = "dl1/calibration/pedestal" # TBD + required_columns = frozenset(["time", "pedestal"]) # TBD + expected_units = {"pedestal": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate pedestal or gain for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + pedestal : array [float] + interpolated pedestal values + """ + + self._check_interpolators(tel_id) + + pedestal = self._interpolators[tel_id]["pedestal"](time) + return pedestal + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "pedestal" + for the pedestal data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("time") + time = input_table["time"] + pedestal = input_table["pedestal"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["pedestal"] = StepFunction( + time, pedestal, **self.interp_options + ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py new file mode 100644 index 00000000000..930e1e7d73c --- /dev/null +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -0,0 +1,179 @@ +import astropy.units as u +import numpy as np +import pytest +import tables +from astropy.table import Table +from astropy.time import Time + +from ctapipe.io.interpolation import ( + FlatFieldInterpolator, + PedestalInterpolator, + PointingInterpolator, +) + +t0 = Time("2022-01-01T00:00:00") + + +def test_azimuth_switchover(): + """Test pointing interpolation""" + + table = Table( + { + "time": t0 + [0, 1, 2] * u.s, + "azimuth": [359, 1, 3] * u.deg, + "altitude": [60, 61, 62] * u.deg, + }, + ) + + interpolator = PointingInterpolator() + interpolator.add_table(1, table) + + alt, az = interpolator(tel_id=1, time=t0 + 0.5 * u.s) + assert u.isclose(az, 360 * u.deg) + assert u.isclose(alt, 60.5 * u.deg) + + +def test_invalid_input(): + """Test invalid pointing tables raise nice errors""" + + wrong_time = Table( + { + "time": [1, 2, 3] * u.s, + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3] * u.deg, + } + ) + + interpolator = PointingInterpolator() + with pytest.raises(TypeError, match="astropy.time.Time"): + interpolator.add_table(1, wrong_time) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.m, + "altitude": [1, 2, 3] * u.deg, + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + wrong_unit = Table( + { + "time": Time(1.7e9 + np.arange(3), format="unix"), + "azimuth": [1, 2, 3] * u.deg, + "altitude": [1, 2, 3], + } + ) + with pytest.raises(ValueError, match="compatible with 'rad'"): + interpolator.add_table(1, wrong_unit) + + +def test_hdf5(tmp_path): + """Test writing interpolated data to file""" + from ctapipe.io import write_table + + table = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + path = tmp_path / "pointing.h5" + write_table(table, path, "/dl0/monitoring/telescope/pointing/tel_001") + with tables.open_file(path) as h5file: + interpolator = PointingInterpolator(h5file) + alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + +def test_bounds(): + """Test invalid pointing tables raise nice errors""" + + table_pointing = Table( + { + "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, + "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, + "altitude": np.linspace(70.0, 60.0, 6) * u.deg, + }, + ) + + table_pedestal = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + table_flatfield = Table( + { + "time": np.arange(0.0, 10.1, 2.0), + "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) + * u.Unit(), + }, + ) + + interpolator_pointing = PointingInterpolator() + interpolator_pedestal = PedestalInterpolator() + interpolator_flatfield = FlatFieldInterpolator() + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + error_message = "below the interpolation range" + + with pytest.raises(ValueError, match=error_message): + interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) + + with pytest.raises(ValueError, match=error_message): + interpolator_pedestal(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match=error_message): + interpolator_flatfield(tel_id=1, time=-0.1) + + with pytest.raises(ValueError, match="above the interpolation range"): + interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 1 * u.s) + assert u.isclose(alt, 69 * u.deg) + assert u.isclose(az, 1 * u.deg) + + pedestal = interpolator_pedestal(tel_id=1, time=1.0) + assert all(pedestal == table_pedestal["pedestal"][0]) + flatfield = interpolator_flatfield(tel_id=1, time=1.0) + assert all(flatfield == table_flatfield["gain"][0]) + with pytest.raises(KeyError): + interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) + with pytest.raises(KeyError): + interpolator_pedestal(tel_id=2, time=1.0) + with pytest.raises(KeyError): + interpolator_flatfield(tel_id=2, time=1.0) + + interpolator_pointing = PointingInterpolator(bounds_error=False) + interpolator_pedestal = PedestalInterpolator(bounds_error=False) + interpolator_flatfield = FlatFieldInterpolator(bounds_error=False) + interpolator_pointing.add_table(1, table_pointing) + interpolator_pedestal.add_table(1, table_pedestal) + interpolator_flatfield.add_table(1, table_flatfield) + + for dt in (-0.1, 10.1) * u.s: + alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) + assert np.isnan(alt.value) + assert np.isnan(az.value) + + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) + interpolator_pointing.add_table(1, table_pointing) + alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) + assert u.isclose(alt, 71 * u.deg) + assert u.isclose(az, -1 * u.deg) + + alt, az = interpolator_pointing(tel_id=1, time=t0 + 11 * u.s) + assert u.isclose(alt, 59 * u.deg) + assert u.isclose(az, 11 * u.deg) From 1afef1e9de713e730863ab4014496d817c31426e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:50:51 +0200 Subject: [PATCH 201/221] Fixed some issues with the ChunkFunction --- src/ctapipe/calib/camera/calibrator.py | 3 +- src/ctapipe/io/interpolation.py | 62 ++++++++++++++++------- src/ctapipe/io/tests/test_interpolator.py | 9 +++- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 8d9e309c077..bcc05ee3c71 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -7,7 +7,7 @@ import astropy.units as u import numpy as np -import Vizier +import Vizier # discuss this dependency with max etc. from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 @@ -251,7 +251,6 @@ def _get_slice_range( return slice_start, slice_stop - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py index 3b792c2107d..e0e27470c99 100644 --- a/src/ctapipe/io/interpolation.py +++ b/src/ctapipe/io/interpolation.py @@ -12,12 +12,13 @@ from .astropy_helpers import read_table -class StepFunction: +class ChunkFunction: """ - Step function Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the closest previous - point given. + Chunk Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the latest starting + valid chunk is given or the earliest available still valid chunk for any + pixels without valid data. Parameters ---------- @@ -31,7 +32,8 @@ class StepFunction: def __init__( self, - times, + start_times, + end_times, values, bounds_error=True, fill_value="extrapolate", @@ -39,12 +41,13 @@ def __init__( copy=False, ): self.values = values - self.times = times + self.start_times = start_times + self.end_times = end_times self.bounds_error = bounds_error self.fill_value = fill_value def __call__(self, point): - if point < self.times[0]: + if point < self.start_times[0]: if self.bounds_error: raise ValueError("below the interpolation range") @@ -56,9 +59,28 @@ def __call__(self, point): a[:] = np.nan return a + elif point > self.end_times[-1]: + if self.bounds_error: + raise ValueError("above the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[-1] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + else: - i = np.searchsorted(self.times, point, side="left") - return self.values[i - 1] + i = np.searchsorted( + self.start_times, point, side="left" + ) # Latest valid chunk + j = np.searchsorted( + self.end_times, point, side="left" + ) # Earliest valid chunk + return np.where( + np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] + ) # Give value for latest chunk unless its nan. If nan give earliest chunk value class Interpolator(Component, metaclass=ABCMeta): @@ -237,7 +259,7 @@ class FlatFieldInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["time", "gain"]) # TBD + required_columns = frozenset(["start_time", "end_time", "gain"]) expected_units = {"gain": u.one} def __call__(self, tel_id, time): @@ -279,12 +301,13 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] gain = input_table["gain"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = StepFunction( - time, gain, **self.interp_options + self._interpolators[tel_id]["gain"] = ChunkFunction( + start_time, end_time, gain, **self.interp_options ) @@ -294,7 +317,7 @@ class PedestalInterpolator(Interpolator): """ telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["time", "pedestal"]) # TBD + required_columns = frozenset(["start_time", "end_time", "pedestal"]) expected_units = {"pedestal": u.one} def __call__(self, tel_id, time): @@ -336,10 +359,11 @@ def add_table(self, tel_id, input_table): self._check_tables(input_table) input_table = input_table.copy() - input_table.sort("time") - time = input_table["time"] + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] pedestal = input_table["pedestal"] self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = StepFunction( - time, pedestal, **self.interp_options + self._interpolators[tel_id]["pedestal"] = ChunkFunction( + start_time, end_time, pedestal, **self.interp_options ) diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py index 930e1e7d73c..20f5657c1ae 100644 --- a/src/ctapipe/io/tests/test_interpolator.py +++ b/src/ctapipe/io/tests/test_interpolator.py @@ -103,7 +103,8 @@ def test_bounds(): table_pedestal = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -111,7 +112,8 @@ def test_bounds(): table_flatfield = Table( { - "time": np.arange(0.0, 10.1, 2.0), + "start_time": np.arange(0.0, 10.1, 2.0), + "end_time": np.arange(0.5, 10.6, 2.0), "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) * u.Unit(), }, @@ -168,6 +170,9 @@ def test_bounds(): assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) + assert all(np.isnan(interpolator_pedestal(tel_id=1, time=20.0))) + assert all(np.isnan(interpolator_flatfield(tel_id=1, time=20.0))) + interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) interpolator_pointing.add_table(1, table_pointing) alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) From ccaffe2c42ecac18aef75b292a73e8daa0aae2af Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:53:05 +0200 Subject: [PATCH 202/221] Adding the StatisticsExtractors --- src/ctapipe/calib/camera/extractor.py | 96 +++++++++++-------- .../calib/camera/tests/test_extractors.py | 84 ++++++++++++++++ 2 files changed, 142 insertions(+), 38 deletions(-) create mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index 3075b33a9ca..bf1b2fdfa01 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -8,15 +8,13 @@ "SigmaClippingExtractor", ] - from abc import abstractmethod import numpy as np -import scipy.stats from astropy.stats import sigma_clipped_stats -from ctapipe.core import TelescopeComponent from ctapipe.containers import StatisticsContainer +from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( Int, List, @@ -24,7 +22,6 @@ class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) image_median_cut_outliers = List( [-0.3, 0.3], @@ -47,7 +44,9 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) @abstractmethod - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: """ Call the relevant functions to extract the statistics for the particular extractor. @@ -66,31 +65,40 @@ def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> List of extracted statistics and validity ranges """ + class PlainExtractor(StatisticsExtractor): """ Extractor the statistics from a sequence of images using numpy and scipy functions """ - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._plain_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._plain_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _plain_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - + def _plain_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_images, axis=0) @@ -102,7 +110,7 @@ def _plain_extraction(self, images, times, masked_pixels_of_sample) -> Statistic pixel_std = np.ma.std(masked_images, axis=0) # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) + # median_of_pixel_median = np.ma.median(pixel_median, axis=1) # outliers from median image_median_outliers = np.logical_or( @@ -135,28 +143,35 @@ class SigmaClippingExtractor(StatisticsExtractor): help="Number of iterations for the sigma clipping outlier removal", ).tag(config=True) - def __call__(self, dl1_table, masked_pixels_of_sample=None, col_name="image") -> list: - + def __call__( + self, dl1_table, masked_pixels_of_sample=None, col_name="image" + ) -> list: # in python 3.12 itertools.batched can be used - image_chunks = (dl1_table[col_name].data[i:i + self.sample_size] for i in range(0, len(dl1_table[col_name].data), self.sample_size)) - time_chunks = (dl1_table["time"][i:i + self.sample_size] for i in range(0, len(dl1_table["time"]), self.sample_size)) + image_chunks = ( + dl1_table[col_name].data[i : i + self.sample_size] + for i in range(0, len(dl1_table[col_name].data), self.sample_size) + ) + time_chunks = ( + dl1_table["time"][i : i + self.sample_size] + for i in range(0, len(dl1_table["time"]), self.sample_size) + ) # Calculate the statistics from a sequence of images stats_list = [] - for images, times in zip(image_chunks,time_chunks): - stats_list.append(self._sigmaclipping_extraction(images, times, masked_pixels_of_sample)) + for images, times in zip(image_chunks, time_chunks): + stats_list.append( + self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) + ) return stats_list - def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> StatisticsContainer: - + def _sigmaclipping_extraction( + self, images, times, masked_pixels_of_sample + ) -> StatisticsContainer: # ensure numpy array - masked_images = np.ma.array( - images, - mask=masked_pixels_of_sample - ) + masked_images = np.ma.array(images, mask=masked_pixels_of_sample) # median of the event images - image_median = np.ma.median(masked_images, axis=-1) + # image_median = np.ma.median(masked_images, axis=-1) # mean, median, and std over the sample per pixel max_sigma = self.sigma_clipping_max_sigma @@ -176,7 +191,7 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) # only warn for values discard in the sigma clipping, not those from before - outliers = unused_values & (~masked_images.mask) + # outliers = unused_values & (~masked_images.mask) # add outliers identified by sigma clipping for following operations masked_images.mask |= unused_values @@ -192,16 +207,21 @@ def _sigmaclipping_extraction(self, images, times, masked_pixels_of_sample) -> S # outliers from median image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - - image_median_outliers = ( - np.logical_or(image_deviation < self.image_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis], - image_deviation > self.image_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis])) + image_median_outliers = np.logical_or( + image_deviation + < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], + image_deviation + > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], + ) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = ( - np.logical_or(deviation < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) + image_std_outliers = np.logical_or( + deviation + < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], + deviation + > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], + ) return StatisticsContainer( validity_start=times[0], diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py new file mode 100644 index 00000000000..a83c93fd1c0 --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_extractors.py @@ -0,0 +1,84 @@ +""" +Tests for StatisticsExtractor and related functions +""" + +import numpy as np +import pytest +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor + + +@pytest.fixture(name="test_plainextractor") +def fixture_test_plainextractor(example_subarray): + """test the PlainExtractor""" + return PlainExtractor(subarray=example_subarray, chunk_size=2500) + + +@pytest.fixture(name="test_sigmaclippingextractor") +def fixture_test_sigmaclippingextractor(example_subarray): + """test the SigmaClippingExtractor""" + return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) + + +def test_extractors(test_plainextractor, test_sigmaclippingextractor): + """test basic functionality of the StatisticsExtractors""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + + pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + + plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) + + assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) + assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) + + +def test_check_outliers(test_sigmaclippingextractor): + """test detection ability of outliers""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_data[:, 0, 120] = 120.0 + flatfield_dl1_data[:, 1, 67] = 120.0 + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table + ) + + # check if outliers where detected correctly + assert sigmaclipping_stats_list[0].median_outliers[0][120] + assert sigmaclipping_stats_list[0].median_outliers[1][67] + assert sigmaclipping_stats_list[1].median_outliers[0][120] + assert sigmaclipping_stats_list[1].median_outliers[1][67] + + +def test_check_chunk_shift(test_sigmaclippingextractor): + """test the chunk shift option and the boundary case for the last chunk""" + + times = np.linspace(60117.911, 60117.9258, num=5000) + flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) + # insert outliers + flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) + sigmaclipping_stats_list = test_sigmaclippingextractor( + dl1_table=flatfield_dl1_table, chunk_shift=2000 + ) + + # check if three chunks are used for the extraction + assert len(sigmaclipping_stats_list) == 3 From 18fb810a1830db63de34e10740f441d4de26ee99 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 3 Sep 2024 15:53:10 +0200 Subject: [PATCH 203/221] I added a basic star fitter --- src/ctapipe/calib/camera/calibrator.py | 209 +-------- src/ctapipe/calib/camera/pointing.py | 571 +++++++++++++++++++++++++ src/ctapipe/containers.py | 29 ++ 3 files changed, 604 insertions(+), 205 deletions(-) create mode 100644 src/ctapipe/calib/camera/pointing.py diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index bcc05ee3c71..6ad47b525fe 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,40 +2,29 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ +<<<<<<< HEAD +======= +>>>>>>> c5f385f0 (I added a basic star fitter) from functools import cache import astropy.units as u import numpy as np -import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, - Float, - Int, - Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "CameraCalibrator", -] +__all__ = ["CameraCalibrator"] @cache def _get_pixel_index(n_pixels): @@ -61,196 +50,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels - -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py new file mode 100644 index 00000000000..2f89159bcdf --- /dev/null +++ b/src/ctapipe/calib/camera/pointing.py @@ -0,0 +1,571 @@ +""" +Definition of the `CameraCalibrator` class, providing all steps needed to apply +calibration and image extraction, as well as supporting algorithms. +""" + +import copy +from functools import cache + +import astropy.units as u +import numpy as np +import Vizier # discuss this dependency with max etc. +from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.table import QTable + +from ctapipe.calib.camera.extractor import StatisticsExtractor +from ctapipe.containers import StarContainer +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.core import TelescopeComponent +from ctapipe.core.traits import ( + ComponentName, + Dict, + Float, + Integer, + TelescopeParameter, +) +from ctapipe.image import tailcuts_clean +from ctapipe.image.psf_model import PSFModel +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader + +__all__ = [ + "PointingCalculator", +] + + +@cache +def _get_pixel_index(n_pixels): + """Cached version of ``np.arange(n_pixels)``""" + return np.arange(n_pixels) + + +def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channel): + broken_pixels = np.zeros((n_channels, n_pixels), dtype=bool) + + index = _get_pixel_index(n_pixels) + masks = ( + pixel_status.hardware_failing_pixels, + pixel_status.pedestal_failing_pixels, + pixel_status.flatfield_failing_pixels, + ) + for mask in masks: + if mask is not None: + if selected_gain_channel is not None: + broken_pixels |= mask[selected_gain_channel, index] + else: + broken_pixels |= mask + + return broken_pixels + + +def cart2pol(x, y): + """ + Convert cartesian coordinates to polar + + :param float x: X coordinate [m] + :param float y: Y coordinate [m] + + :return: Tuple (r, φ)[m, rad] + """ + rho = np.sqrt(x**2 + y**2) + phi = np.arctan2(y, x) + return (rho, phi) + + +def pol2cart(rho, phi): + """ + Convert polar coordinates to cartesian + + :param float rho: R coordinate + :param float phi: ¢ coordinate [rad] + + :return: Tuple (x,y)[m, m] + """ + x = rho * np.cos(phi) + y = rho * np.sin(phi) + return (x, y) + + +class PointingCalculator(TelescopeComponent): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + cleaning = Dict( + {"bound_thresh": 750, "pic_thresh": 15000}, help="Image cleaning parameters" + ).tag(config=True) + + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + + psf_model_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="ComaModel", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__( + subarray=subarray, + stats_extractor="Plain", + config=config, + parent=parent, + **kwargs, + ) + + self.psf = PSFModel.from_name( + self.psf_model_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, input_url, tel_id): + self.tel_id = tel_id + + if self._check_req_data(input_url, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + # first get the camera geometry and pointing for the file and determine what stars we should see + + with EventSource(input_url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) # get some pointing to make a list of stars that we expect to see + + self.pointing = self.pointing.transform_to("icrs") + + self.broken_pixels = np.unique(np.where(self.broken_pixels)) + + self.image_size = len( + event.variance_image.image + ) # get the size of images of the camera we are calibrating + + self.stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] # get all stars that could be in the fov + + self.stars_in_fov = self.stars_in_fov[ + self.tars_in_fov["Bmag"] < self.max_star_magnitude + ] # select stars for magnitude to exclude those we would not be able to see + + # get the accumulated variance images + + ( + accumulated_pointing, + accumulated_times, + variance_statistics, + ) = self._get_accumulated_images(input_url) + + accumulated_images = np.array([x.mean for x in variance_statistics]) + + star_pixels = self._get_expected_star_pixels( + accumulated_times, accumulated_pointing + ) + + star_mask = np.ones(self.image_size, dtype=bool) + + star_mask[star_pixels] = False + + # get NSB values + + nsb = np.mean(accumulated_images[star_mask], axis=1) + nsb_std = np.std(accumulated_images[star_mask], axis=1) + + clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) + + reco_stars = [] + + for i, image in enumerate(clean_images): + reco_stars.append([]) + camera_frame = EngineeringCameraFrame( + telescope_pointing=accumulated_pointing[i], + focal_length=self.focal_length, + obstime=accumulated_times[i].utc, + location=self.location, + ) + for star in self.stars_in_fov: + reco_stars[-1].append( + self._fit_star_position( + star, accumulated_times[i], camera_frame, image, nsb_std[i] + ) + ) + + return reco_stars + + # now fit the star locations + + def _check_req_data(self, url, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[self.tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + + def _calibrate_var_images(self, var_images, time, calibration_file): + """ + Calibrate a set of variance images + + Parameters + ---------- + var_images : list + list of variance images + time : list + list of times correxponding to the variance images + calibration_file : str + name of the file where the calibration data can be found + """ + # So i need to use the interpolator classes to read the calibration data + relative_gains = FlatFieldInterpolator( + calibration_file + ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, + np.square(relative_gains(time[i])), + ) + + return var_images + + def _get_expected_star_pixels(self, time_list, pointing_list): + """ + Determine which in which pixels stars are expected for a series of images + + Parameters + ---------- + time_list : list + list of time values where the images were capturedd + pointing_list : list + list of pointing values for the images + """ + + res = [] + + for pointing, time in zip( + pointing_list, time_list + ): # loop over time and pointing of images + temp = [] + + camera_frame = EngineeringCameraFrame( + telescope_pointing=pointing, + focal_length=self.focal_length, + obstime=time.utc, + location=self.location, + ) # get the engineering camera frame for the pointing + + for star in self.stars_in_fov: + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + expected_central_pixel = self.camera_geometry.transform_to( + camera_frame + ).position_to_pix_index( + star_coords.x, star_coords.y + ) # get where the star should be + cluster = copy.deepcopy( + self.camera_geometry.neighbors[expected_central_pixel] + ) # get the neighborhood of the star + cluster_corona = [] + + for pixel_index in cluster: + cluster_corona.extend( + copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) + ) # and add another layer of pixels to be sure + + cluster.extend(cluster_corona) + cluster.append(expected_central_pixel) + temp.extend(list(set(cluster))) + + res.append(temp) + + return res + + def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + star_coords = SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + star_coords = star_coords.transform_to(camera_frame) + + rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) + + if phi < 0: + phi = phi + 2 * np.pi + + star_container = StarContainer( + label=star["NOMAD1"], + magnitude=star["Bmag"], + expected_x=star_coords.x, + expected_y=star_coords.y, + expected_r=rho * u.m, + expected_phi=phi * u.rad, + timestamp=timestamp, + ) + + current_geometry = self.camera_geometry.transform_to(camera_frame) + + hit_pdf = self._get_star_pdf(star, current_geometry) + cluster = np.where(hit_pdf > self.pdf_percentile_limit * np.sum(hit_pdf)) + + if not np.any(image[cluster] > self.min_star_prominence * nsb_std): + self.log.info("Star %s can not be detected", star["NOMAD1"]) + star.pixels = np.full(self.max_cluster_size, -1) + return star_container + + pad_size = self.max_cluster_size - len(cluster[0]) + if pad_size > 0: + star.pixels = np.pad(cluster[0], (0, pad_size), constant_values=-1) + else: + star.pixels = cluster[0][: self.max_cluster_size] + + self.log.warning( + "Reconstructed cluster is longer than %s, truncated cluster info will " + "be recorded to the output table. Not a big deal, as correct cluster " + "used for position reconstruction.", + self.max_cluster_size, + ) + return star_container + + rs, fs = cart2pol( + current_geometry.pix_x[cluster].to_value(u.m), + current_geometry.pix_y[cluster].to_value(u.m), + ) + + k, r0, sr = self.psf_model.radial_pdf_params + + star_container.reco_r = ( + self.coma_r_shift_correction + * np.average(rs, axis=None, weights=image[cluster], returned=False) + * u.m + ) + + star_container.reco_x = self.coma_r_shift_correction * np.average( + current_geometry.pix_x[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + star_container.reco_y = self.coma_r_shift_correction * np.average( + current_geometry.pix_y[cluster], + axis=None, + weights=image[cluster], + returned=False, + ) + + _, star_container.reco_phi = cart2pol(star.reco_x, star.reco_y) + + if star_container.reco_phi < 0: + star_container.reco_phi = star.reco_phi + 2 * np.pi * u.rad + + star_container.reco_dx = ( + np.sqrt(np.cov(current_geometry.pix_x[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dy = ( + np.sqrt(np.cov(current_geometry.pix_y[cluster], aweights=hit_pdf[cluster])) + * u.m + ) + + star_container.reco_dr = np.sqrt(np.cov(rs, aweights=hit_pdf[cluster])) * u.m + + _, star_container.reco_dphi = cart2pol( + star_container.reco_dx, star_container.reco_dy + ) + + return star_container + + def _get_accumulated_images(self, input_url): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=self.tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=True, + ) + + # get the trigger type for all images and make a mask + + event_mask = dl1_table["event_type"] == 2 + + # get the pointing for all images and filter for trigger type + + altitude = dl1_table["telescope_pointing_altitude"][event_mask] + azimuth = dl1_table["telescope_pointing_azimuth"][event_mask] + time = dl1_table["time"][event_mask] + + pointing = [ + SkyCoord( + az=x, alt=y, frame="altaz", obstime=z.tai.utc, location=self.location + ) + for x, y, z in zip(azimuth, altitude, time) + ] + + # get the time and images from the data + + variance_images = copy.deepcopy(dl1_table["variance_image"][event_mask]) + + # now make a filter to to reject EAS light and starlight and keep a separate EAS filter + + charge_images = dl1_table["image"][event_mask] + + light_mask = [ + tailcuts_clean( + self.camera_geometry, + x, + picture_thresh=self.cleaning["pic_thresh"], + boundary_thresh=self.cleaning["bound_thresh"], + ) + for x in charge_images + ] + + shower_mask = copy.deepcopy(light_mask) + + star_pixels = self._get_expected_star_pixels(time, pointing) + + light_mask[:, star_pixels] = True + + if self.broken_pixels is not None: + light_mask[:, self.broken_pixels] = True + + # calculate the average variance in viable pixels and replace the values where there is EAS light + + mean_variance = np.mean(variance_images[~light_mask]) + + variance_images[shower_mask] = mean_variance + + # now calibrate the images + + variance_images = self._calibrate_var_images( + self, variance_images, time, input_url + ) + + # Get the average variance across the data to + + # then turn it into a table that the extractor can read + variance_image_table = QTable([time, variance_images], names=["time", "image"]) + + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[self.tel_id]] + + # get the cumulative variance images using the statistics extractor and return the value + + variance_statistics = extractor(variance_image_table) + + accumulated_times = np.array([x.validity_start for x in variance_statistics]) + + # calculate where stars might be + + accumulated_pointing = np.array( + [x for x in pointing if pointing.time in accumulated_times] + ) + + return (accumulated_pointing, accumulated_times, variance_statistics) + + def _get_star_pdf(self, star, current_geometry): + image = np.zeros(self.image_size) + + r0 = star.expected_r.to_value(u.m) + f0 = star.expected_phi.to_value(u.rad) + + self.psf_model.update_model_parameters(r0, f0) + + dr = ( + self.pdf_bin_size + * np.rad2deg(np.arctan(1 / self.focal_length.to_value(u.m))) + / 3600.0 + ) + r = np.linspace( + r0 - dr * self.n_pdf_bins / 2.0, + r0 + dr * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + df = np.deg2rad(self.pdf_bin_size / 3600.0) * 100 + f = np.linspace( + f0 - df * self.n_pdf_bins / 2.0, + f0 + df * self.n_pdf_bins / 2.0, + self.n_pdf_bins, + ) + + for r_ in r: + for f_ in f: + val = self.psf_model.pdf(r_, f_) * dr * df + x, y = pol2cart(r_, f_) + pixelN = current_geometry.position_to_pix_index(x * u.m, y * u.m) + if pixelN != -1: + image[pixelN] += val + + return image diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index a937f49d337..a706d50627d 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -64,6 +64,7 @@ "ObservationBlockContainer", "ObservingMode", "ObservationBlockState", + "StarContainer", ] @@ -1540,3 +1541,31 @@ class ObservationBlockContainer(Container): scheduled_start_time = Field(NAN_TIME, "expected start time from scheduler") actual_start_time = Field(NAN_TIME, "true start time") actual_duration = Field(nan * u.min, "true duration", unit=u.min) + + +class StarContainer(Container): + "Stores information about a star in the field of view of a camera." + + label = Field("", "Star label", dtype=np.str_) + magnitude = Field(-1, "Star magnitude") + expected_x = Field(np.nan * u.m, "Expected star position (x)", unit=u.m) + expected_y = Field(np.nan * u.m, "Expected star position (y)", unit=u.m) + + expected_r = Field(np.nan * u.m, "Expected star position (r)", unit=u.m) + expected_phi = Field(np.nan * u.rad, "Expected star position (phi)", unit=u.rad) + + reco_x = Field(np.nan * u.m, "Reconstructed star position (x)", unit=u.m) + reco_y = Field(np.nan * u.m, "Reconstructed star position (y)", unit=u.m) + reco_dx = Field(np.nan * u.m, "Reconstructed star position error (x)", unit=u.m) + reco_dy = Field(np.nan * u.m, "Reconstructed star position error (y)", unit=u.m) + + reco_r = Field(np.nan * u.m, "Reconstructed star position (r)", unit=u.m) + reco_phi = Field(np.nan * u.rad, "Reconstructed star position (phi)", unit=u.rad) + reco_dr = Field(np.nan * u.m, "Reconstructed star position error (r)", unit=u.m) + reco_dphi = Field( + np.nan * u.rad, "Reconstructed star position error (phi)", unit=u.rad + ) + + timestamp = Field(NAN_TIME, "Reconstruction timestamp") + + pixels = Field(np.full(20, -1), "List of star pixel ids", dtype=np.int_, ndim=1) From 89a30ef946a5fda3a4e7809d2cb4630d7746230c Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 12 Sep 2024 11:08:20 +0200 Subject: [PATCH 204/221] Added the fitting --- src/ctapipe/calib/camera/pointing.py | 326 ++++++++++++++++++++++++++- 1 file changed, 322 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 2f89159bcdf..77105194382 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -8,9 +8,12 @@ import astropy.units as u import numpy as np +import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord from astropy.table import QTable +from astropy.time import Time +from scipy.odr import ODR, Model, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer @@ -85,6 +88,87 @@ def pol2cart(rho, phi): return (x, y) +class StarTracker: + """ + Utility class to provide the position of the star in the telescope's camera frame coordinates at a given time + """ + + def __init__( + self, + star_label, + star_coordinates, + telescope_location, + telescope_focal_length, + telescope_pointing, + observed_wavelength, + relative_humidity, + temperature, + pressure, + pointing_label=None, + ): + """ + Constructor + + :param str star_label: Star label + :param SkyCoord star_coordinates: Star coordinates in ICRS frame + :param EarthLocation telescope_location: Telescope location coordinates + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str pointing_label: Pointing label + """ + self.star_label = star_label + self.star_coordinates_icrs = star_coordinates + self.telescope_location = telescope_location + self.telescope_focal_length = telescope_focal_length + self.telescope_pointing = telescope_pointing + self.obswl = observed_wavelength + self.relative_humidity = relative_humidity + self.temperature = temperature + self.pressure = pressure + self.pointing_label = pointing_label + + def position_in_camera_frame(self, timestamp, pointing=None, focal_correction=0): + """ + Calculates star position in the engineering camera frame + + :param astropy.Time timestamp: Timestamp of the observation + :param SkyCoord pointing: Current telescope pointing in ICRS frame + :param float focal_correction: Correction to the focal length of the telescope. Float, should be provided in meters + + :return: Pair (float, float) of star's (x,y) coordinates in the engineering camera frame in meters + """ + # If no telescope pointing is provided, use the telescope pointing, provided + # during the class member initialization + if pointing is None: + pointing = self.telescope_pointing + # Determine current telescope pointing in AltAz + altaz_pointing = pointing.transform_to( + AltAz( + obstime=timestamp, + location=self.telescope_location, + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + ) + # Create current camera frame + camera_frame = EngineeringCameraFrame( + telescope_pointing=altaz_pointing, + focal_length=self.telescope_focal_length + focal_correction * u.m, + obstime=timestamp, + location=self.telescope_location, + ) + # Calculate the star's coordinates in the current camera frame + star_coords_camera = self.star_coordinates_icrs.transform_to(camera_frame) + return (star_coords_camera.x.to_value(), star_coords_camera.y.to_value()) + + class PointingCalculator(TelescopeComponent): """ Component to calculate pointing corrections from interleaved skyfield events. @@ -103,6 +187,12 @@ class PointingCalculator(TelescopeComponent): "elevation - in meters", ).tag(config=True) + observed_wavelength = Float( + 0.35, + help="Observed star light wavelength in microns" + "(convolution of blackbody spectrum with camera sensitivity)", + ).tag(config=True) + min_star_prominence = Integer( 3, help="Minimal star prominence over the background in terms of " @@ -128,6 +218,11 @@ class PointingCalculator(TelescopeComponent): help="Name of the PSFModel Subclass to be used.", ).tag(config=True) + meteo_parameters = Dict( + {"relative_humidity": 0.5, "temperature": 10, "pressure": 790}, + help="Meteorological parameters in [dimensionless, deg C, hPa]", + ).tag(config=True) + def __init__( self, subarray, @@ -196,6 +291,8 @@ def __call__(self, input_url, tel_id): self.tars_in_fov["Bmag"] < self.max_star_magnitude ] # select stars for magnitude to exclude those we would not be able to see + star_labels = [x.label for x in self.stars_in_fov] + # get the accumulated variance images ( @@ -240,7 +337,17 @@ def __call__(self, input_url, tel_id): return reco_stars - # now fit the star locations + # now fit the pointing correction. + fitter = Pointing_Fitter( + star_labels, + self.pointing, + self.location, + self.focal_length, + self.observed_wavelength, + self.meteo_parameters, + ) + + fitter.fit(accumulated_pointing) def _check_req_data(self, url, calibration_type): """ @@ -299,7 +406,7 @@ def _get_expected_star_pixels(self, time_list, pointing_list): Parameters ---------- time_list : list - list of time values where the images were capturedd + list of time values where the images were captured pointing_list : list list of pointing values for the images """ @@ -346,10 +453,13 @@ def _get_expected_star_pixels(self, time_list, pointing_list): return res - def _fit_star_position(self, star, timestamp, camera_frame, image, nsb_std): + def _fit_star_position( + self, star, timestamp, camera_frame, image, nsb_std, current_pointing + ): star_coords = SkyCoord( star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" ) + star_coords = star_coords.transform_to(camera_frame) rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) @@ -569,3 +679,211 @@ def _get_star_pdf(self, star, current_geometry): image[pixelN] += val return image + + +class Pointing_Fitter: + """ + Pointing correction fitter + """ + + def __init__( + self, + stars, + times, + telescope_pointing, + telescope_location, + focal_length, + observed_wavelength, + meteo_params, + fit_grid="polar", + ): + """ + Constructor + + :param list stars: List of lists of star containers the first dimension is supposed to be time + :param list time: List of time values for the when the star locations were fitted + :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame + :param EarthLocation telescope_location: Telescope location + :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] + :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] + :param float relative_humidity: Relative humidity + :param Quantity[u.deg_C] temperature: Temperature [C] + :param Quantity[u.hPa] pressure: Pressure [hPa] + :param str fit_grid: Coordinate system grid to use. Either polar or cartesian + """ + self.star_containers = stars + self.times = times + self.telescope_pointing = telescope_pointing + self.telescope_location = telescope_location + self.focal_length = focal_length + self.obswl = observed_wavelength + self.relative_humidity = meteo_params["relative_humidity"] + self.temperature = meteo_params["temperature"] + self.pressure = meteo_params["pressure"] + self.stars = [] + self.visible = [] + self.data = [] + self.errors = [] + # Construct the data here. Stars that were not found are marked in the variable "visible" and use the coordinates (0,0) whenever they can not be seen + for star_list in stars: + self.data.append([]) + self.errors.append([]) + self.visible.append({}) + for star in star_list: + if star.reco_x != np.nan * u.m: + self.visible[-1].update({star.label: True}) + self.data[-1].append(star.reco_x) + self.data[-1].append(star.reco_y) + self.errors[-1].append(star.reco_dx) + self.errors[-1].append(star.reco_dy) + else: + self.visible[-1].update({star.label: False}) + self.data[-1].append(0) + self.data[-1].append(0) + self.errors[-1].append( + 1000.0 + ) # large error value to suppress the stars that were not found + self.errors[-1].append(1000.0) + + for star in stars[0]: + self.stars.append(self.init_star(star.label)) + self.fit_mode = "xy" + self.fit_grid = fit_grid + self.star_motion_model = Model(self.fit_function) + self.fit_summary = None + self.fit_resuts = None + + def init_star(self, star_label): + """ + Initialize StarTracker object for a given star + + :param str star_label: Star label according to NOMAD catalog + + :return: StarTracker object + """ + star = Vizier(catalog="NOMAD").query_constraints(NOMAD1=star_label)[0] + star_coords = SkyCoord( + star["RAJ2000"], + star["DEJ2000"], + unit="deg", + frame="icrs", + obswl=self.obswl, + relative_humidity=self.relative_humidity, + temperature=self.temperature, + pressure=self.pressure, + ) + st = StarTracker( + star_label, + star_coords, + self.telescope_location, + self.focal_length, + self.telescope_pointing, + self.obswl, + self.relative_humidity, + self.temperature, + self.pressure, + ) + return st + + def current_pointing(self, t): + """ + Retrieve current telescope pointing + """ + index = self.times.index(t) + + return self.telescope_pointing[index] + + def fit_function(self, p, t): + """ + Construct the fit function for the pointing correction + + p: Fit parameters + t: Timestamp in UNIX_TAI format + + """ + + time = Time(t, format="unix_tai", scale="utc") + index = self.times.index(time) + coord_list = [] + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + + m_ra, m_dec = p + new_ra = self.current_pointing(time).ra + m_ra * u.deg + new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) + for star in self.stars: + if self.visible[index][star.label]: # if star was visible give the value + x, y = star.position_in_camera_frame(time, new_pointing) + else: # otherwise set it to (0,0) and set a large error value + x, y = (0, 0) + if self.fit_grid == "polar": + x, y = cart2pol(x, y) + coord_list.extend([x]) + coord_list.extend([y]) + + return coord_list + + def fit(self, data, errors, time_range, fit_mode="xy"): + """ + Performs the ODR fit of stars trajectories and saves the results as self.fit_results + + :param array data: Reconstructed star positions, data.shape = (N(stars) * 2, len(time_range)), order: x_1, y_1...x_N, y_N + :param array errors: Uncertainties on the reconstructed star positions. Same shape and order as for the data + :param array time_range: Array of timestamps in UNIX_TAI format + :param array-like(SkyCoord) pointings: Array of telescope pointings in ICRS frame + :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. + """ + self.fit_mode = fit_mode + if self.fit_mode == "radec" or self.fit_mode == "xy": + init_mispointing = [0, 0] + elif self.fit_mode == "y": + init_mispointing = [0] + elif self.fit_mode == "xyz": + init_mispointing = [0, 0, 0] + if errors is not None: + rdata = RealData(x=self.times, y=data, sy=errors) + else: + rdata = RealData(x=self.times, y=data) + odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) + self.fit_summary = odr.run() + if self.fit_mode == "radec": + self.fit_results = pd.DataFrame( + data={ + "dRA": [self.fit_summary.beta[0]], + "dDEC": [self.fit_summary.beta[1]], + "eRA": [self.fit_summary.sd_beta[0]], + "eDEC": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "xy": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + } + ) + elif self.fit_mode == "y": + self.fit_results = pd.DataFrame( + data={ + "dY": [self.fit_summary.beta[0]], + "eY": [self.fit_summary.sd_beta[0]], + } + ) + elif self.fit_mode == "xyz": + self.fit_results = pd.DataFrame( + data={ + "dX": [self.fit_summary.beta[0]], + "dY": [self.fit_summary.beta[1]], + "dZ": [self.fit_summary.beta[2]], + "eX": [self.fit_summary.sd_beta[0]], + "eY": [self.fit_summary.sd_beta[1]], + "eZ": [self.fit_summary.sd_beta[2]], + } + ) From f22de63a7684739c65b0b74598f0f84a99696f0c Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 20 Sep 2024 14:31:30 +0200 Subject: [PATCH 205/221] Finihed V1 of the startracker code --- src/ctapipe/calib/camera/pointing.py | 90 +++++++++++----------------- 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 77105194382..c530eacdfb4 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -291,8 +291,6 @@ def __call__(self, input_url, tel_id): self.tars_in_fov["Bmag"] < self.max_star_magnitude ] # select stars for magnitude to exclude those we would not be able to see - star_labels = [x.label for x in self.stars_in_fov] - # get the accumulated variance images ( @@ -335,19 +333,22 @@ def __call__(self, input_url, tel_id): ) ) - return reco_stars + # now fit the pointing corrections. + correction = [] + for i, stars in enumerate(reco_stars): + fitter = Pointing_Fitter( + stars, + accumulated_times[i], + accumulated_pointing[i], + self.location, + self.focal_length, + self.observed_wavelength, + self.meteo_parameters, + ) - # now fit the pointing correction. - fitter = Pointing_Fitter( - star_labels, - self.pointing, - self.location, - self.focal_length, - self.observed_wavelength, - self.meteo_parameters, - ) + correction.append(fitter.fit()) - fitter.fit(accumulated_pointing) + return correction def _check_req_data(self, url, calibration_type): """ @@ -689,7 +690,7 @@ class Pointing_Fitter: def __init__( self, stars, - times, + time, telescope_pointing, telescope_location, focal_length, @@ -712,7 +713,7 @@ def __init__( :param str fit_grid: Coordinate system grid to use. Either polar or cartesian """ self.star_containers = stars - self.times = times + self.time = time self.telescope_pointing = telescope_pointing self.telescope_location = telescope_location self.focal_length = focal_length @@ -721,32 +722,16 @@ def __init__( self.temperature = meteo_params["temperature"] self.pressure = meteo_params["pressure"] self.stars = [] - self.visible = [] self.data = [] self.errors = [] - # Construct the data here. Stars that were not found are marked in the variable "visible" and use the coordinates (0,0) whenever they can not be seen - for star_list in stars: - self.data.append([]) - self.errors.append([]) - self.visible.append({}) - for star in star_list: - if star.reco_x != np.nan * u.m: - self.visible[-1].update({star.label: True}) - self.data[-1].append(star.reco_x) - self.data[-1].append(star.reco_y) - self.errors[-1].append(star.reco_dx) - self.errors[-1].append(star.reco_dy) - else: - self.visible[-1].update({star.label: False}) - self.data[-1].append(0) - self.data[-1].append(0) - self.errors[-1].append( - 1000.0 - ) # large error value to suppress the stars that were not found - self.errors[-1].append(1000.0) - - for star in stars[0]: - self.stars.append(self.init_star(star.label)) + # Construct the data here. Add only stars that were found + for star in stars: + if not np.isnan(star.reco_x): + self.data.append(star.reco_x) + self.data.append(star.reco_y) + self.errors.append(star.reco_dx) + self.errors.append(star.reco_dy) + self.stars.append(self.init_star(star.label)) self.fit_mode = "xy" self.fit_grid = fit_grid self.star_motion_model = Model(self.fit_function) @@ -803,24 +788,20 @@ def fit_function(self, p, t): """ time = Time(t, format="unix_tai", scale="utc") - index = self.times.index(time) coord_list = [] m_ra, m_dec = p - new_ra = self.current_pointing(time).ra + m_ra * u.deg - new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_ra = self.current_pointing(t).ra + m_ra * u.deg + new_dec = self.current_pointing(t).dec + m_dec * u.deg new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) m_ra, m_dec = p - new_ra = self.current_pointing(time).ra + m_ra * u.deg - new_dec = self.current_pointing(time).dec + m_dec * u.deg + new_ra = self.current_pointing(t).ra + m_ra * u.deg + new_dec = self.current_pointing(t).dec + m_dec * u.deg new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) for star in self.stars: - if self.visible[index][star.label]: # if star was visible give the value - x, y = star.position_in_camera_frame(time, new_pointing) - else: # otherwise set it to (0,0) and set a large error value - x, y = (0, 0) + x, y = star.position_in_camera_frame(time, new_pointing) if self.fit_grid == "polar": x, y = cart2pol(x, y) coord_list.extend([x]) @@ -828,14 +809,10 @@ def fit_function(self, p, t): return coord_list - def fit(self, data, errors, time_range, fit_mode="xy"): + def fit(self, fit_mode="xy"): """ Performs the ODR fit of stars trajectories and saves the results as self.fit_results - :param array data: Reconstructed star positions, data.shape = (N(stars) * 2, len(time_range)), order: x_1, y_1...x_N, y_N - :param array errors: Uncertainties on the reconstructed star positions. Same shape and order as for the data - :param array time_range: Array of timestamps in UNIX_TAI format - :param array-like(SkyCoord) pointings: Array of telescope pointings in ICRS frame :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. """ self.fit_mode = fit_mode @@ -845,10 +822,10 @@ def fit(self, data, errors, time_range, fit_mode="xy"): init_mispointing = [0] elif self.fit_mode == "xyz": init_mispointing = [0, 0, 0] - if errors is not None: - rdata = RealData(x=self.times, y=data, sy=errors) + if self.errors is not None: + rdata = RealData(x=self.time, y=self.data, sy=self.errors) else: - rdata = RealData(x=self.times, y=data) + rdata = RealData(x=self.time, y=self.data) odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) self.fit_summary = odr.run() if self.fit_mode == "radec": @@ -887,3 +864,4 @@ def fit(self, data, errors, time_range, fit_mode="xy"): "eZ": [self.fit_summary.sd_beta[2]], } ) + return self.fit_results From 23e9b25eaf19d561ce57b4ca41a5f05ea8eaf61e Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 14:45:27 +0200 Subject: [PATCH 206/221] I separated the star tracking out to the StarTracer class --- src/ctapipe/calib/camera/pointing.py | 918 ++++++++++++--------------- src/ctapipe/image/psf_model.py | 50 +- 2 files changed, 436 insertions(+), 532 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index c530eacdfb4..933105efeb6 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -10,10 +10,9 @@ import numpy as np import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord +from astropy.coordinates import Angle, EarthLocation, SkyCoord from astropy.table import QTable -from astropy.time import Time -from scipy.odr import ODR, Model, RealData +from scipy.odr import ODR, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer @@ -28,11 +27,10 @@ ) from ctapipe.image import tailcuts_clean from ctapipe.image.psf_model import PSFModel -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader +from ctapipe.instrument import CameraGeometry +from ctapipe.io import FlatFieldInterpolator, PointingInterpolator -__all__ = [ - "PointingCalculator", -] +__all__ = ["PointingCalculator", "StarImageGenerator"] @cache @@ -60,6 +58,19 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +def get_index_step(val, lookup): + index = 0 + + for i, x in enumerate(lookup): + if val <= x: + index = i + + if val > lookup[-1]: + index = len(lookup) - 1 + + return index + + def cart2pol(x, y): """ Convert cartesian coordinates to polar @@ -88,85 +99,258 @@ def pol2cart(rho, phi): return (x, y) -class StarTracker: +def get_star_pdf(r0, f0, geometry, psf, n_pdf_bins, pdf_bin_size, focal_length): + image = np.zeros(len(geometry)) + + psf.update_model_parameters(r0, f0) + + dr = pdf_bin_size * np.rad2deg(np.arctan(1 / focal_length)) / 3600.0 + r = np.linspace( + r0 - dr * n_pdf_bins / 2.0, + r0 + dr * n_pdf_bins / 2.0, + n_pdf_bins, + ) + df = np.deg2rad(pdf_bin_size / 3600.0) * 100 + f = np.linspace( + f0 - df * n_pdf_bins / 2.0, + f0 + df * n_pdf_bins / 2.0, + n_pdf_bins, + ) + + for r_ in r: + for f_ in f: + val = psf.pdf(r_, f_) * dr * df + x, y = pol2cart(r_, f_) + pixelN = geometry.position_to_pix_index(x * u.m, y * u.m) + if pixelN != -1: + image[pixelN] += val + + return image + + +def StarImageGenerator( + self, + radius, + phi, + magnitude, + n_pdf_bins, + pdf_bin_size, + psf_model_name, + psf_model_pars, + camera_name, + focal_length, +): """ - Utility class to provide the position of the star in the telescope's camera frame coordinates at a given time + :param list stars: list of star containers, stars to be placed in image + :param dict psf_model_pars: psf model parameters + """ + camera_geometry = CameraGeometry.from_name(camera_name) + psf = PSFModel.from_name(self.psf_model_type, subarray=self.subarray, parent=self) + psf.update_model_parameters(psf_model_pars) + image = np.zeros(len(camera_geometry)) + for r, p, m in zip(radius, phi, magnitude): + image += m * get_star_pdf( + r, p, camera_geometry, psf, n_pdf_bins, pdf_bin_size, focal_length + ) + + return image + + +class StarTracer: + """ + Utility class to trace a set of stars over a period of time and generate their locations in the camera """ def __init__( self, - star_label, - star_coordinates, - telescope_location, - telescope_focal_length, - telescope_pointing, + stars, + magnitude, + az, + alt, + time, + meteo_parameters, observed_wavelength, - relative_humidity, - temperature, - pressure, - pointing_label=None, + camera_geometry, + focal_length, + location, ): """ - Constructor - - :param str star_label: Star label - :param SkyCoord star_coordinates: Star coordinates in ICRS frame - :param EarthLocation telescope_location: Telescope location coordinates - :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] - :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame - :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] - :param float relative_humidity: Relative humidity - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.hPa] pressure: Pressure [hPa] - :param str pointing_label: Pointing label + param dict stars: dict of Astropy.SkyCoord objects, keys are the nomad labels + param dict magnitude: + param list time: list of Astropy.time objects corresponding to the altitude and azimuth values """ - self.star_label = star_label - self.star_coordinates_icrs = star_coordinates - self.telescope_location = telescope_location - self.telescope_focal_length = telescope_focal_length - self.telescope_pointing = telescope_pointing - self.obswl = observed_wavelength - self.relative_humidity = relative_humidity - self.temperature = temperature - self.pressure = pressure - self.pointing_label = pointing_label - - def position_in_camera_frame(self, timestamp, pointing=None, focal_correction=0): + + self.stars = stars + self.magnitude = magnitude + + self.pointing = PointingInterpolator() + pointing = QTable([az, alt, time], names=["azimuth", "altitude", "time"]) + self.pointing.add_table(0, pointing) + + self.meteo_parameters = meteo_parameters + self.observed_wavelength = observed_wavelength + self.camera_geometry = camera_geometry + self.focal_length = focal_length * u.m + self.location = location + + @classmethod + def from_lookup( + cls, + max_star_magnitude, + az, + alt, + time, + meteo_params, + observed_wavelength, + camera_geometry, + focal_length, + location, + ): + """ + classmethod to use vizier lookup to generate a class instance """ - Calculates star position in the engineering camera frame + _pointing = SkyCoord( + az=az[0], + alt=alt[0], + frame="altaz", + obstime=time[0], + location=location, + obswl=observed_wavelength * u.micron, + relative_humidity=meteo_params["relative_humidity"], + temperature=meteo_params["temperature"] * u.deg_C, + pressure=meteo_params["pressure"] * u.hPa, + ) + stars_in_fov = Vizier.query_region( + _pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] # get all stars that could be in the fov - :param astropy.Time timestamp: Timestamp of the observation - :param SkyCoord pointing: Current telescope pointing in ICRS frame - :param float focal_correction: Correction to the focal length of the telescope. Float, should be provided in meters + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < max_star_magnitude] - :return: Pair (float, float) of star's (x,y) coordinates in the engineering camera frame in meters + stars = {} + magnitude = {} + for star in stars_in_fov: + star_coords = { + star["NOMAD1"]: SkyCoord( + star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" + ) + } + stars.update(star_coords) + magnitude.update({star["NOMAD1"]: star["Bmag"]}) + + return cls( + stars, + magnitude, + az, + alt, + time, + meteo_params, + observed_wavelength, + camera_geometry, + focal_length, + location, + ) + + def get_star_labels(self): """ - # If no telescope pointing is provided, use the telescope pointing, provided - # during the class member initialization - if pointing is None: - pointing = self.telescope_pointing - # Determine current telescope pointing in AltAz - altaz_pointing = pointing.transform_to( - AltAz( - obstime=timestamp, - location=self.telescope_location, - obswl=self.obswl, - relative_humidity=self.relative_humidity, - temperature=self.temperature, - pressure=self.pressure, - ) + Return a list of all stars that are being traced + """ + + return list(self.stars.keys()) + + def get_magnitude(self, star): + """ + Return the magnitude of star + + parameter str star: NOMAD1 label of the star + """ + + return self.magnitude[star] + + def get_pointing(self, t, offset=(0.0, 0.0)): + alt, az = self.pointing(t) + alt += offset[0] * u.rad + az += offset[1] * u.rad + + coords = SkyCoord( + az=az, + alt=alt, + frame="altaz", + obstime=t, + location=self.location, + obswl=self.observed_wavelength * u.micron, + relative_humidity=self.meteo_parameters["relative_humidity"], + temperature=self.meteo_parameters["temperature"] * u.deg_C, + pressure=self.meteo_parameters["pressure"] * u.hPa, ) - # Create current camera frame + + return coords + + def get_camera_frame(self, t, offset=(0.0, 0.0), focal_correction=0.0): + altaz_pointing = self.get_pointing(t, offset=offset) + camera_frame = EngineeringCameraFrame( telescope_pointing=altaz_pointing, - focal_length=self.telescope_focal_length + focal_correction * u.m, - obstime=timestamp, - location=self.telescope_location, + focal_length=self.focal_length + focal_correction * u.m, + obstime=t, + location=self.location, ) - # Calculate the star's coordinates in the current camera frame - star_coords_camera = self.star_coordinates_icrs.transform_to(camera_frame) - return (star_coords_camera.x.to_value(), star_coords_camera.y.to_value()) + + return camera_frame + + def get_current_geometry(self, t, offset=(0.0, 0.0), focal_correction=0.0): + camera_frame = self.get_camera_frame( + t, offset=offset, focal_correction=focal_correction + ) + + current_geometry = self.camera_geometry.transform_to(camera_frame) + + return current_geometry + + def get_position_in_camera(self, t, star, offset=(0.0, 0.0), focal_correction=0.0): + camera_frame = self.get_camera_frame( + t, offset=offset, focal_correction=focal_correction + ) + # Calculate the stars coordinates in the current camera frame + coords = self.stars[star].transform_to(camera_frame) + return (coords.x.to_value(), coords.y.to_value()) + + def get_position_in_pixel(self, t, star, focal_correction=0.0): + x, y = self.get_position_in_camera(t, star, focal_correction=focal_correction) + current_geometry = self.get_current_geometry(t) + + return current_geometry.position_to_pix_index(x, y) + + def get_expected_star_pixels(self, t, focal_correction=0.0): + """ + Determine which in which pixels stars are expected for a series of images + + Parameters + ---------- + t : list + list of time values where the images were captured + """ + + res = [] + + for star in self.get_star_labels(): + expected_central_pixel = self.get_positions_in_pixel( + t, star, focal_correction=focal_correction + ) + cluster = copy.deepcopy( + self.camera_geometry.neighbors[expected_central_pixel] + ) # get the neighborhood of the star + cluster_corona = [] + + for pixel_index in cluster: + cluster_corona.extend( + copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) + ) # and add another layer of pixels to be sure + + cluster.extend(cluster_corona) + cluster.append(expected_central_pixel) + res.extend(list(set(cluster))) + + return res class PointingCalculator(TelescopeComponent): @@ -181,6 +365,12 @@ class PointingCalculator(TelescopeComponent): The location of the telescope for which the pointing correction is to be calculated """ + stats_extractor = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="Plain"), + default_value="Plain", + help="Name of the StatisticsExtractor Subclass to be used.", + ).tag(config=True) + telescope_location = Dict( {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, help="Telescope location, longitude and latitude should be expressed in deg, " @@ -200,7 +390,7 @@ class PointingCalculator(TelescopeComponent): ).tag(config=True) max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + 7.0, help="Maximal magnitude of the star to be considered in the analysis" ).tag(config=True) cleaning = Dict( @@ -213,7 +403,7 @@ class PointingCalculator(TelescopeComponent): ).tag(config=True) psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + trait=ComponentName(PSFModel, default_value="ComaModel"), default_value="ComaModel", help="Name of the PSFModel Subclass to be used.", ).tag(config=True) @@ -223,16 +413,22 @@ class PointingCalculator(TelescopeComponent): help="Meteorological parameters in [dimensionless, deg C, hPa]", ).tag(config=True) + n_pdf_bins = Integer(1000, help="Camera focal length").tag(config=True) + + pdf_bin_size = Float(10.0, help="Camera focal length").tag(config=True) + + focal_length = Float(1.0, help="Camera focal length in meters").tag(config=True) + def __init__( self, subarray, + geometry, config=None, parent=None, **kwargs, ): super().__init__( subarray=subarray, - stats_extractor="Plain", config=config, parent=parent, **kwargs, @@ -247,240 +443,110 @@ def __init__( lat=self.telescope_location["latitude"] * u.deg, height=self.telescope_location["elevation"] * u.m, ) - - def __call__(self, input_url, tel_id): - self.tel_id = tel_id - - if self._check_req_data(input_url, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - # first get the camera geometry and pointing for the file and determine what stars we should see - - with EventSource(input_url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) # get some pointing to make a list of stars that we expect to see - - self.pointing = self.pointing.transform_to("icrs") - - self.broken_pixels = np.unique(np.where(self.broken_pixels)) - - self.image_size = len( - event.variance_image.image - ) # get the size of images of the camera we are calibrating - - self.stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] # get all stars that could be in the fov - - self.stars_in_fov = self.stars_in_fov[ - self.tars_in_fov["Bmag"] < self.max_star_magnitude - ] # select stars for magnitude to exclude those we would not be able to see - - # get the accumulated variance images - - ( - accumulated_pointing, - accumulated_times, - variance_statistics, - ) = self._get_accumulated_images(input_url) - - accumulated_images = np.array([x.mean for x in variance_statistics]) - - star_pixels = self._get_expected_star_pixels( - accumulated_times, accumulated_pointing + self.stats_aggregator = StatisticsExtractor.from_name( + self.stats_extractor, subarray=self.subarray, parent=self ) - star_mask = np.ones(self.image_size, dtype=bool) - - star_mask[star_pixels] = False - - # get NSB values - - nsb = np.mean(accumulated_images[star_mask], axis=1) - nsb_std = np.std(accumulated_images[star_mask], axis=1) - - clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) - - reco_stars = [] - - for i, image in enumerate(clean_images): - reco_stars.append([]) - camera_frame = EngineeringCameraFrame( - telescope_pointing=accumulated_pointing[i], - focal_length=self.focal_length, - obstime=accumulated_times[i].utc, - location=self.location, - ) - for star in self.stars_in_fov: - reco_stars[-1].append( - self._fit_star_position( - star, accumulated_times[i], camera_frame, image, nsb_std[i] - ) - ) + self.set_camera(geometry) - # now fit the pointing corrections. - correction = [] - for i, stars in enumerate(reco_stars): - fitter = Pointing_Fitter( - stars, - accumulated_times[i], - accumulated_pointing[i], - self.location, - self.focal_length, - self.observed_wavelength, - self.meteo_parameters, - ) + def set_camera(self, geometry, focal_lengh): + if isinstance(geometry, str): + self.camera_geometry = CameraGeometry.from_name(geometry) - correction.append(fitter.fit()) + self.pixel_radius = self.camera_geometry.pixel_width[0] - return correction - - def _check_req_data(self, url, calibration_type): + def ingest_data(self, data_table): """ - Check if the prerequisite calibration data exists in the files - - Parameters + Attributes ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain + data_table : Table + Table containing a series of variance images with corresponding initial pointing values, trigger times and calibration data """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - calibration_data = getattr(event.mon.tel[self.tel_id], calibration_type) - - if calibration_data is None: - return False - - return True + # set up the StarTracer here to track stars in the camera + self.tracer = StarTracer.from_lookup( + data_table["telescope_pointing_azimuth"], + data_table["telescope_pointing_altitude"], + data_table["time"], + self.meteo_parameters, + self.observed_wavelength, + self.camera_geometry, + self.focal_length, + self.telescope_location, + ) - def _calibrate_var_images(self, var_images, time, calibration_file): - """ - Calibrate a set of variance images + self.broken_pixels = np.unique(np.where(data_table["unusable_pixels"])) - Parameters - ---------- - var_images : list - list of variance images - time : list - list of times correxponding to the variance images - calibration_file : str - name of the file where the calibration data can be found - """ - # So i need to use the interpolator classes to read the calibration data - relative_gains = FlatFieldInterpolator( - calibration_file - ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square(relative_gains(time[i])), + for azimuth, altitude, time in zip( + data_table["telescope_pointing_azimuth"], + data_table["telescope_pointing_altitude"], + data_table["time"], + ): + _pointing = SkyCoord( + az=azimuth, + alt=altitude, + frame="altaz", + obstime=time, + location=self.location, + obswl=self.observed_wavelength * u.micron, + relative_humidity=self.meteo_parameters["relative_humidity"], + temperature=self.meteo_parameters["temperature"] * u.deg_C, + pressure=self.meteo_parameters["pressure"] * u.hPa, ) + self.pointing.append(_pointing) - return var_images + self.image_size = len( + data_table["variance_images"][0].image + ) # get the size of images of the camera we are calibrating - def _get_expected_star_pixels(self, time_list, pointing_list): - """ - Determine which in which pixels stars are expected for a series of images + # get the accumulated variance images - Parameters - ---------- - time_list : list - list of time values where the images were captured - pointing_list : list - list of pointing values for the images - """ + self._get_accumulated_images(data_table) - res = [] + def fit_stars(self): + stars = self.tracer.get_star_labels() - for pointing, time in zip( - pointing_list, time_list - ): # loop over time and pointing of images - temp = [] + self.all_containers = [] - camera_frame = EngineeringCameraFrame( - telescope_pointing=pointing, - focal_length=self.focal_length, - obstime=time.utc, - location=self.location, - ) # get the engineering camera frame for the pointing + for t, image in (self.accumulated_times, self.accumulated_images): + self.all_containers.append([]) - for star in self.stars_in_fov: - star_coords = SkyCoord( - star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" - ) - star_coords = star_coords.transform_to(camera_frame) - expected_central_pixel = self.camera_geometry.transform_to( - camera_frame - ).position_to_pix_index( - star_coords.x, star_coords.y - ) # get where the star should be - cluster = copy.deepcopy( - self.camera_geometry.neighbors[expected_central_pixel] - ) # get the neighborhood of the star - cluster_corona = [] - - for pixel_index in cluster: - cluster_corona.extend( - copy.deepcopy(self.camera_geometry.neighbors[pixel_index]) - ) # and add another layer of pixels to be sure - - cluster.extend(cluster_corona) - cluster.append(expected_central_pixel) - temp.extend(list(set(cluster))) - - res.append(temp) + for star in stars: + container = self._fit_star_position(star, t, image, self.nsb_std) - return res + self.all_containers[-1].append(container) - def _fit_star_position( - self, star, timestamp, camera_frame, image, nsb_std, current_pointing - ): - star_coords = SkyCoord( - star["RAJ2000"], star["DEJ2000"], unit="deg", frame="icrs" - ) + return self.all_containers - star_coords = star_coords.transform_to(camera_frame) + def _fit_star_position(self, star, t, image, nsb_std): + x, y = self.tracer.get_position_in_camera(self, t, star) - rho, phi = cart2pol(star_coords.x.to_value(u.m), star_coords.y.to_value(u.m)) + rho, phi = cart2pol(x, y) if phi < 0: phi = phi + 2 * np.pi star_container = StarContainer( - label=star["NOMAD1"], - magnitude=star["Bmag"], - expected_x=star_coords.x, - expected_y=star_coords.y, + label=star, + magnitude=self.tracer.get_magnitude(star), + expected_x=x, + expected_y=y, expected_r=rho * u.m, expected_phi=phi * u.rad, - timestamp=timestamp, + timestamp=t, ) - current_geometry = self.camera_geometry.transform_to(camera_frame) + current_geometry = self.tracer.get_current_geometry(t) - hit_pdf = self._get_star_pdf(star, current_geometry) + hit_pdf = get_star_pdf( + rho, + phi, + current_geometry, + self.psf, + self.n_pdf_bins, + self.pdf_bin_size, + self.focal_length.to_value(u.m), + ) cluster = np.where(hit_pdf > self.pdf_percentile_limit * np.sum(hit_pdf)) if not np.any(image[cluster] > self.min_star_prominence * nsb_std): @@ -552,47 +618,11 @@ def _fit_star_position( return star_container - def _get_accumulated_images(self, input_url): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=self.tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=True, - ) - - # get the trigger type for all images and make a mask - - event_mask = dl1_table["event_type"] == 2 - - # get the pointing for all images and filter for trigger type - - altitude = dl1_table["telescope_pointing_altitude"][event_mask] - azimuth = dl1_table["telescope_pointing_azimuth"][event_mask] - time = dl1_table["time"][event_mask] - - pointing = [ - SkyCoord( - az=x, alt=y, frame="altaz", obstime=z.tai.utc, location=self.location - ) - for x, y, z in zip(azimuth, altitude, time) - ] - - # get the time and images from the data - - variance_images = copy.deepcopy(dl1_table["variance_image"][event_mask]) + def _get_accumulated_images(self, data_table): + variance_images = data_table["variance_images"] # now make a filter to to reject EAS light and starlight and keep a separate EAS filter - charge_images = dl1_table["image"][event_mask] - light_mask = [ tailcuts_clean( self.camera_geometry, @@ -600,12 +630,14 @@ def _get_accumulated_images(self, input_url): picture_thresh=self.cleaning["pic_thresh"], boundary_thresh=self.cleaning["bound_thresh"], ) - for x in charge_images + for x in data_table["charge_image"] ] shower_mask = copy.deepcopy(light_mask) - star_pixels = self._get_expected_star_pixels(time, pointing) + star_pixels = [ + self.tracer.get_expected_star_pixels(t) for t in data_table["time"] + ] light_mask[:, star_pixels] = True @@ -620,163 +652,46 @@ def _get_accumulated_images(self, input_url): # now calibrate the images - variance_images = self._calibrate_var_images( - self, variance_images, time, input_url - ) + relative_gains = FlatFieldInterpolator() + relative_gains.add_table(0, data_table["relative_gain"]) - # Get the average variance across the data to + for i, var_image in enumerate(variance_images): + variance_images[i] = np.divide( + var_image, + np.square(relative_gains(data_table["time"][i]).median), + ) # then turn it into a table that the extractor can read - variance_image_table = QTable([time, variance_images], names=["time", "image"]) - - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[self.tel_id]] + variance_image_table = QTable( + [data_table["time"], variance_images], names=["time", "image"] + ) # get the cumulative variance images using the statistics extractor and return the value - variance_statistics = extractor(variance_image_table) - - accumulated_times = np.array([x.validity_start for x in variance_statistics]) - - # calculate where stars might be - - accumulated_pointing = np.array( - [x for x in pointing if pointing.time in accumulated_times] + variance_statistics = self.stats_aggregator( + variance_image_table, col_name="image" ) - return (accumulated_pointing, accumulated_times, variance_statistics) - - def _get_star_pdf(self, star, current_geometry): - image = np.zeros(self.image_size) - - r0 = star.expected_r.to_value(u.m) - f0 = star.expected_phi.to_value(u.rad) - - self.psf_model.update_model_parameters(r0, f0) - - dr = ( - self.pdf_bin_size - * np.rad2deg(np.arctan(1 / self.focal_length.to_value(u.m))) - / 3600.0 - ) - r = np.linspace( - r0 - dr * self.n_pdf_bins / 2.0, - r0 + dr * self.n_pdf_bins / 2.0, - self.n_pdf_bins, + self.accumulated_times = np.array( + [x.validity_start for x in variance_statistics] ) - df = np.deg2rad(self.pdf_bin_size / 3600.0) * 100 - f = np.linspace( - f0 - df * self.n_pdf_bins / 2.0, - f0 + df * self.n_pdf_bins / 2.0, - self.n_pdf_bins, - ) - - for r_ in r: - for f_ in f: - val = self.psf_model.pdf(r_, f_) * dr * df - x, y = pol2cart(r_, f_) - pixelN = current_geometry.position_to_pix_index(x * u.m, y * u.m) - if pixelN != -1: - image[pixelN] += val - - return image + accumulated_images = np.array([x.mean for x in variance_statistics]) -class Pointing_Fitter: - """ - Pointing correction fitter - """ + star_pixels = [ + self.tracer.get_expected_star_pixels(t) for t in data_table["time"] + ] - def __init__( - self, - stars, - time, - telescope_pointing, - telescope_location, - focal_length, - observed_wavelength, - meteo_params, - fit_grid="polar", - ): - """ - Constructor - - :param list stars: List of lists of star containers the first dimension is supposed to be time - :param list time: List of time values for the when the star locations were fitted - :param SkyCoord telescope_pointing: Telescope pointing in ICRS frame - :param EarthLocation telescope_location: Telescope location - :param Quantity[u.m] telescope_focal_length: Telescope focal length [m] - :param Quantity[u.micron] observed_wavelength: Telescope focal length [micron] - :param float relative_humidity: Relative humidity - :param Quantity[u.deg_C] temperature: Temperature [C] - :param Quantity[u.hPa] pressure: Pressure [hPa] - :param str fit_grid: Coordinate system grid to use. Either polar or cartesian - """ - self.star_containers = stars - self.time = time - self.telescope_pointing = telescope_pointing - self.telescope_location = telescope_location - self.focal_length = focal_length - self.obswl = observed_wavelength - self.relative_humidity = meteo_params["relative_humidity"] - self.temperature = meteo_params["temperature"] - self.pressure = meteo_params["pressure"] - self.stars = [] - self.data = [] - self.errors = [] - # Construct the data here. Add only stars that were found - for star in stars: - if not np.isnan(star.reco_x): - self.data.append(star.reco_x) - self.data.append(star.reco_y) - self.errors.append(star.reco_dx) - self.errors.append(star.reco_dy) - self.stars.append(self.init_star(star.label)) - self.fit_mode = "xy" - self.fit_grid = fit_grid - self.star_motion_model = Model(self.fit_function) - self.fit_summary = None - self.fit_resuts = None - - def init_star(self, star_label): - """ - Initialize StarTracker object for a given star + star_mask = np.ones(self.image_size, dtype=bool) - :param str star_label: Star label according to NOMAD catalog + star_mask[star_pixels] = False - :return: StarTracker object - """ - star = Vizier(catalog="NOMAD").query_constraints(NOMAD1=star_label)[0] - star_coords = SkyCoord( - star["RAJ2000"], - star["DEJ2000"], - unit="deg", - frame="icrs", - obswl=self.obswl, - relative_humidity=self.relative_humidity, - temperature=self.temperature, - pressure=self.pressure, - ) - st = StarTracker( - star_label, - star_coords, - self.telescope_location, - self.focal_length, - self.telescope_pointing, - self.obswl, - self.relative_humidity, - self.temperature, - self.pressure, - ) - return st + # get NSB values - def current_pointing(self, t): - """ - Retrieve current telescope pointing - """ - index = self.times.index(t) + nsb = np.mean(accumulated_images[star_mask], axis=1) + self.nsb_std = np.std(accumulated_images[star_mask], axis=1) - return self.telescope_pointing[index] + self.clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) def fit_function(self, p, t): """ @@ -787,81 +702,48 @@ def fit_function(self, p, t): """ - time = Time(t, format="unix_tai", scale="utc") coord_list = [] - m_ra, m_dec = p - new_ra = self.current_pointing(t).ra + m_ra * u.deg - new_dec = self.current_pointing(t).dec + m_dec * u.deg - - new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) - - m_ra, m_dec = p - new_ra = self.current_pointing(t).ra + m_ra * u.deg - new_dec = self.current_pointing(t).dec + m_dec * u.deg - new_pointing = SkyCoord(ICRS(ra=new_ra, dec=new_dec)) - for star in self.stars: - x, y = star.position_in_camera_frame(time, new_pointing) - if self.fit_grid == "polar": - x, y = cart2pol(x, y) - coord_list.extend([x]) - coord_list.extend([y]) + index = get_index_step( + t, self.accumulated_times + ) # this gives you the index corresponding to the + for star in self.all_containers[index]: + if not np.isnan(star.reco_x): + x, y = self.tracer.get_position_in_camera(star.label, t, offset=p) + coord_list.extend([x]) + coord_list.extend([y]) return coord_list - def fit(self, fit_mode="xy"): + def fit(self): """ Performs the ODR fit of stars trajectories and saves the results as self.fit_results :param str fit_mode: Fit mode. Can be 'y', 'xy' (default), 'xyz' or 'radec'. """ - self.fit_mode = fit_mode - if self.fit_mode == "radec" or self.fit_mode == "xy": + + results = [] + for i, t in enumerate(self.accumulated_times): init_mispointing = [0, 0] - elif self.fit_mode == "y": - init_mispointing = [0] - elif self.fit_mode == "xyz": - init_mispointing = [0, 0, 0] - if self.errors is not None: - rdata = RealData(x=self.time, y=self.data, sy=self.errors) - else: - rdata = RealData(x=self.time, y=self.data) - odr = ODR(rdata, self.star_motion_model, beta0=init_mispointing) - self.fit_summary = odr.run() - if self.fit_mode == "radec": - self.fit_results = pd.DataFrame( - data={ - "dRA": [self.fit_summary.beta[0]], - "dDEC": [self.fit_summary.beta[1]], - "eRA": [self.fit_summary.sd_beta[0]], - "eDEC": [self.fit_summary.sd_beta[1]], - } - ) - elif self.fit_mode == "xy": - self.fit_results = pd.DataFrame( - data={ - "dX": [self.fit_summary.beta[0]], - "dY": [self.fit_summary.beta[1]], - "eX": [self.fit_summary.sd_beta[0]], - "eY": [self.fit_summary.sd_beta[1]], - } - ) - elif self.fit_mode == "y": - self.fit_results = pd.DataFrame( - data={ - "dY": [self.fit_summary.beta[0]], - "eY": [self.fit_summary.sd_beta[0]], - } - ) - elif self.fit_mode == "xyz": - self.fit_results = pd.DataFrame( + data = [] + errors = [] + for star in self.all_containers: + if not np.isnan(star.reco_x): + data.append(star.reco_x) + data.append(star.reco_y) + errors.append(star.reco_dx) + errors.append(star.reco_dy) + + rdata = RealData(x=[t], y=data, sy=self.errors) + odr = ODR(rdata, self.fit_function, beta0=init_mispointing) + fit_summary = odr.run() + fit_results = pd.DataFrame( data={ - "dX": [self.fit_summary.beta[0]], - "dY": [self.fit_summary.beta[1]], - "dZ": [self.fit_summary.beta[2]], - "eX": [self.fit_summary.sd_beta[0]], - "eY": [self.fit_summary.sd_beta[1]], - "eZ": [self.fit_summary.sd_beta[2]], + "dAZ": [fit_summary.beta[0]], + "dALT": [fit_summary.beta[1]], + "eAZ": [fit_summary.sd_beta[0]], + "eALT": [fit_summary.sd_beta[1]], } ) - return self.fit_results + results.append(fit_results) + return results diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 458070b8145..4bea11fd464 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -8,7 +8,6 @@ import numpy as np from scipy.stats import laplace, laplace_asymmetric -from traitlets import List class PSFModel: @@ -39,6 +38,10 @@ def from_name(cls, name, **kwargs): def pdf(self, *args): pass + @abstractmethod + def update_location(self, *args): + pass + @abstractmethod def update_model_parameters(self, *args): pass @@ -49,18 +52,23 @@ class ComaModel(PSFModel): PSF model, describing pure coma aberrations PSF effect """ - asymmetry_params = List( - default_value=[0.49244797, 9.23573115, 0.15216096], - help="Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera", - ).tag(config=True) - radial_scale_params = List( - default_value=[0.01409259, 0.02947208, 0.06000271, -0.02969355], - help="Parameters describing the dependency of the radial scale on the distance to the center of the camera", - ).tag(config=True) - az_scale_params = List( - default_value=[0.24271557, 7.5511501, 0.02037972], - help="Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera", - ).tag(config=True) + def __init__( + self, + asymmetry_params=[0.49244797, 9.23573115, 0.15216096], + radial_scale_params=[0.01409259, 0.02947208, 0.06000271, -0.02969355], + az_scale_params=[0.24271557, 7.5511501, 0.02037972], + ): + """ + PSF model, describing pure coma aberrations PSF effect + + param list asymmetry_params Parameters describing the dependency of the asymmetry of the psf on the distance to the center of the camera + param list radial_scale_params Parameters describing the dependency of the radial scale on the distance to the center of the camera + param list radial_scale_params Parameters describing the dependency of the azimuthal scale scale on the distance to the center of the camera + """ + + self.asymmetry_params = asymmetry_params + self.radial_scale_params = radial_scale_params + self.az_scale_params = az_scale_params def k_func(self, x): return ( @@ -87,7 +95,21 @@ def pdf(self, r, f): f, *self.azimuthal_pdf_params ) - def update_model_parameters(self, r, f): + def update_model_parameters(self, model_params): + if not ( + model_params["asymmetry_params"] == 3 + and model_params["radial_scale_params"] == 4 + and model_params["az_scale_params"] == 3 + ): + raise ValueError( + "asymmetry_params and az_scale_params needs to have length 3 and radial_scale_params length 4" + ) + + self.asymmetry_params = model_params["asymmetry_params"] + self.radial_scale_params = model_params["radial_scale_params"] + self.az_scale_params = model_params["az_scale_params"] + + def update_location(self, r, f): k = self.k_func(r) sr = self.sr_func(r) sf = self.sf_func(r) From 8c826ebb5db7427c31d5b39159c5f113fd825143 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 15:00:33 +0200 Subject: [PATCH 207/221] Adding a placeholder test script --- src/ctapipe/calib/camera/tests/test_pointing.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/ctapipe/calib/camera/tests/test_pointing.py diff --git a/src/ctapipe/calib/camera/tests/test_pointing.py b/src/ctapipe/calib/camera/tests/test_pointing.py new file mode 100644 index 00000000000..c29014190cb --- /dev/null +++ b/src/ctapipe/calib/camera/tests/test_pointing.py @@ -0,0 +1,3 @@ +""" +Tests for StatisticsExtractor and related functions +""" From 5a4342ecb26abac644b75ff7032a9bc84d4201cc Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 20 Aug 2024 15:31:20 +0200 Subject: [PATCH 208/221] Copying over code for interpolators and pointing calculators --- src/ctapipe/calib/camera/calibrator.py | 366 ++++++++++++++++++++++++- src/ctapipe/image/psf_model.py | 1 - 2 files changed, 363 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 6ad47b525fe..72f62a8ba72 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,27 +2,43 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ -<<<<<<< HEAD -======= ->>>>>>> c5f385f0 (I added a basic star fitter) +import pickle +from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np +import Vizier +from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 +from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + Dict, + Float, + Int, + Integer, + Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler +from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer +from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader +__all__ = [ + "CalibrationCalculator", + "TwoPassStatisticsCalculator", + "PointingCalculator", + "CameraCalibrator", +] +>>>>>>> b729f829 (Copying over code for interpolators and pointing calculators) __all__ = ["CameraCalibrator"] @@ -50,6 +66,350 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels +class CalibrationCalculator(TelescopeComponent): + """ + Base component for various calibration calculators + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + """ + + stats_extractor_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), + default_value="PlainExtractor", + help="Name of the StatisticsExtractor subclass to be used.", + ).tag(config=True) + + output_path = Path(help="output filename").tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + stats_extractor=None, + **kwargs, + ): + """ + Parameters + ---------- + subarray: ctapipe.instrument.SubarrayDescription + Description of the subarray. Provides information about the + camera which are useful in calibration. Also required for + configuring the TelescopeParameter traitlets. + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + This is mutually exclusive with passing a ``parent``. + parent: ctapipe.core.Component or ctapipe.core.Tool + Parent of this component in the configuration hierarchy, + this is mutually exclusive with passing ``config`` + stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor + The StatisticsExtractor to use. If None, the default via the + configuration system will be constructed. + """ + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + self.subarray = subarray + + self.stats_extractor = {} + + if stats_extractor is None: + for _, _, name in self.stats_extractor_type: + self.stats_extractor[name] = StatisticsExtractor.from_name( + name, subarray=self.subarray, parent=self + ) + else: + name = stats_extractor.__class__.__name__ + self.stats_extractor_type = [("type", "*", name)] + self.stats_extractor[name] = stats_extractor + + @abstractmethod + def __call__(self, input_url, tel_id): + """ + Call the relevant functions to calculate the calibration coefficients + for a given set of events + + Parameters + ---------- + input_url : str + URL where the events are stored from which the calibration coefficients + are to be calculated + tel_id : int + The telescope id + """ + + +class TwoPassStatisticsCalculator(CalibrationCalculator): + """ + Component to calculate statistics from calibration events. + """ + + faulty_pixels_threshold = Float( + 0.1, + help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", + ).tag(config=True) + chunk_shift = Int( + 100, + help="Number of samples to shift the extraction chunk for the calculation of the statistical values", + ).tag(config=True) + + def __call__( + self, + input_url, + tel_id, + col_name="image", + ): + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + # Get the extractor + extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] + + # First pass through the whole provided dl1 data + stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) + + # Second pass + stats_list = [] + faultless_previous_chunk = False + for chunk_nr, stats in enumerate(stats_list_firstpass): + # Append faultless stats from the previous chunk + if faultless_previous_chunk: + stats_list.append(stats_list_firstpass[chunk_nr - 1]) + + # Detect faulty pixels over all gain channels + outlier_mask = np.logical_or( + stats.median_outliers[0], stats.std_outliers[0] + ) + if len(stats.median_outliers) == 2: + outlier_mask = np.logical_or( + outlier_mask, + np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), + ) + # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. + if ( + np.count_nonzero(outlier_mask) / len(outlier_mask) + > self.faulty_pixels_threshold + ): + slice_start, slice_stop = self._get_slice_range( + chunk_nr=chunk_nr, + chunk_size=extractor.chunk_size, + faultless_previous_chunk=faultless_previous_chunk, + last_chunk=len(stats_list_firstpass) - 1, + last_element=len(dl1_table[tel_id]) - 1, + ) + # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. + if slice_stop - slice_start > extractor.chunk_size: + # Slice the dl1 table according to the previously calculated start and stop. + dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] + # Run the stats extractor on the sliced dl1 table with a chunk_shift + # to remove the period of trouble (carflashes etc.) as effectively as possible. + stats_list_secondpass = extractor( + dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift + ) + # Extend the final stats list by the stats list of the second pass. + stats_list.extend(stats_list_secondpass) + else: + # Store the last chunk in case the two last chunks are faulty. + stats_list.append(stats_list_firstpass[chunk_nr]) + # Set the boolean to False to track this chunk as faulty for the next iteration. + faultless_previous_chunk = False + else: + # Set the boolean to True to track this chunk as faultless for the next iteration. + faultless_previous_chunk = True + + # Open the output file and store the final stats list. + with open(self.output_path, "wb") as f: + pickle.dump(stats_list, f) + + def _get_slice_range( + self, + chunk_nr, + chunk_size, + faultless_previous_chunk, + last_chunk, + last_element, + ): + slice_start = 0 + if chunk_nr > 0: + slice_start = ( + chunk_size * (chunk_nr - 1) + self.chunk_shift + if faultless_previous_chunk + else chunk_size * chunk_nr + self.chunk_shift + ) + slice_stop = last_element + if chunk_nr < last_chunk: + slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 + + return slice_start, slice_stop + + +class PointingCalculator(CalibrationCalculator): + """ + Component to calculate pointing corrections from interleaved skyfield events. + + Attributes + ---------- + stats_extractor: str + The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + telescope_location: dict + The location of the telescope for which the pointing correction is to be calculated + """ + + telescope_location = Dict( + {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, + help="Telescope location, longitude and latitude should be expressed in deg, " + "elevation - in meters", + ).tag(config=True) + + min_star_prominence = Integer( + 3, + help="Minimal star prominence over the background in terms of " + "NSB variance std deviations", + ).tag(config=True) + + max_star_magnitude = Float( + 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" + ).tag(config=True) + + psf_model_type = TelescopeParameter( + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), + default_value="ComaModel", + help="Name of the PSFModel Subclass to be used.", + ).tag(config=True) + + def __init__( + self, + subarray, + config=None, + parent=None, + **kwargs, + ): + super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) + + # TODO: Currently not in the dependency list of ctapipe + + self.psf = PSFModel.from_name( + self.pas_model_type, subarray=self.subarray, parent=self + ) + + self.location = EarthLocation( + lon=self.telescope_location["longitude"] * u.deg, + lat=self.telescope_location["latitude"] * u.deg, + height=self.telescope_location["elevation"] * u.m, + ) + + def __call__(self, input_url, tel_id): + if self._check_req_data(input_url, tel_id, "flatfield"): + raise KeyError( + "Relative gain not found. Gain calculation needs to be performed first." + ) + + self.tel_id = tel_id + + # first get thecamera geometry and pointing for the file and determine what stars we should see + + with EventSource(input_url, max_events=1) as src: + self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry + self.focal_length = src.subarray.tel[ + self.tel_id + ].optics.equivalent_focal_length + self.pixel_radius = self.camera_geometry.pixel_width[0] + + event = next(iter(src)) + + self.pointing = SkyCoord( + az=event.pointing.tel[self.telescope_id].azimuth, + alt=event.pointing.tel[self.telescope_id].altitude, + frame="altaz", + obstime=event.trigger.time.utc, + location=self.location, + ) + + stars_in_fov = Vizier.query_region( + self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" + )[0] + + stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] + + # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader + input_data = TableLoader(input_url=input_url) + dl1_table = input_data.read_telescope_events_by_id( + telescopes=tel_id, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=False, + true_parameters=False, + instrument=False, + pointing=False, + ) + + # get the time and images from the data + + variance_images = dl1_table["variance_image"] + + time = dl1_table["time"] + + # now calibrate the images + + variance_images = self._calibrate_var_images( + self, variance_images, time, input_url + ) + + def _check_req_data(self, url, tel_id, calibration_type): + """ + Check if the prerequisite calibration data exists in the files + + Parameters + ---------- + url : str + URL of file that is to be tested + tel_id : int + The telescope id. + calibration_type : str + Name of the field that is to be looked for e.g. flatfield or + gain + """ + with EventSource(url, max_events=1) as source: + event = next(iter(source)) + + calibration_data = getattr(event.mon.tel[tel_id], calibration_type) + + if calibration_data is None: + return False + + return True + + def _calibrate_var_images(self, var_images, time, calibration_file): + # So i need to use the interpolator classes to read the calibration data + relative_gains = FlatFieldInterpolator( + calibration_file + ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance + + for i, var_image in enumerate(var_images): + var_images[i].image = np.divide( + var_image.image, + np.square(relative_gains(time[i])), + ) + + return var_images + + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/image/psf_model.py b/src/ctapipe/image/psf_model.py index 4bea11fd464..1f21dd4ac2c 100644 --- a/src/ctapipe/image/psf_model.py +++ b/src/ctapipe/image/psf_model.py @@ -9,7 +9,6 @@ import numpy as np from scipy.stats import laplace, laplace_asymmetric - class PSFModel: def __init__(self, **kwargs): """ From 675bc9bcab84946041268da47f0630153322dfa9 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:50:51 +0200 Subject: [PATCH 209/221] Fixed some issues with the ChunkFunction --- src/ctapipe/calib/camera/calibrator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 72f62a8ba72..564692bdfe8 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -9,7 +9,7 @@ import astropy.units as u import numpy as np -import Vizier +import Vizier # discuss this dependency with max etc. from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 @@ -299,8 +299,6 @@ def __init__( ): super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - # TODO: Currently not in the dependency list of ctapipe - self.psf = PSFModel.from_name( self.pas_model_type, subarray=self.subarray, parent=self ) From 05aada67b3ed323ea298507215410235fb31afad Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 22 Aug 2024 12:53:05 +0200 Subject: [PATCH 210/221] Adding the StatisticsExtractors --- src/ctapipe/calib/camera/extractor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py index bf1b2fdfa01..7093d057f20 100644 --- a/src/ctapipe/calib/camera/extractor.py +++ b/src/ctapipe/calib/camera/extractor.py @@ -72,7 +72,6 @@ class PlainExtractor(StatisticsExtractor): using numpy and scipy functions """ - def __call__( self, dl1_table, masked_pixels_of_sample=None, col_name="image" ) -> list: From 5c51ed90693dae61800c780d466c47cddeab3fd1 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 3 Sep 2024 15:53:10 +0200 Subject: [PATCH 211/221] I added a basic star fitter --- src/ctapipe/calib/camera/calibrator.py | 363 ------------------------- src/ctapipe/calib/camera/pointing.py | 11 +- 2 files changed, 3 insertions(+), 371 deletions(-) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index 564692bdfe8..cbf5442ed7e 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -2,43 +2,22 @@ Definition of the `CameraCalibrator` class, providing all steps needed to apply calibration and image extraction, as well as supporting algorithms. """ - -import pickle -from abc import abstractmethod from functools import cache import astropy.units as u import numpy as np -import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord from numba import float32, float64, guvectorize, int64 -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - Dict, - Float, - Int, - Integer, - Path, TelescopeParameter, ) from ctapipe.image.extractor import ImageExtractor from ctapipe.image.invalid_pixels import InvalidPixelHandler -from ctapipe.image.psf_model import PSFModel from ctapipe.image.reducer import DataVolumeReducer -from ctapipe.io import EventSource, FlatFieldInterpolator, TableLoader - -__all__ = [ - "CalibrationCalculator", - "TwoPassStatisticsCalculator", - "PointingCalculator", - "CameraCalibrator", -] ->>>>>>> b729f829 (Copying over code for interpolators and pointing calculators) __all__ = ["CameraCalibrator"] @@ -66,348 +45,6 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels -class CalibrationCalculator(TelescopeComponent): - """ - Base component for various calibration calculators - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - """ - - stats_extractor_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="PlainExtractor"), - default_value="PlainExtractor", - help="Name of the StatisticsExtractor subclass to be used.", - ).tag(config=True) - - output_path = Path(help="output filename").tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_extractor=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_extractor: ctapipe.calib.camera.extractor.StatisticsExtractor - The StatisticsExtractor to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - self.stats_extractor = {} - - if stats_extractor is None: - for _, _, name in self.stats_extractor_type: - self.stats_extractor[name] = StatisticsExtractor.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_extractor.__class__.__name__ - self.stats_extractor_type = [("type", "*", name)] - self.stats_extractor[name] = stats_extractor - - @abstractmethod - def __call__(self, input_url, tel_id): - """ - Call the relevant functions to calculate the calibration coefficients - for a given set of events - - Parameters - ---------- - input_url : str - URL where the events are stored from which the calibration coefficients - are to be calculated - tel_id : int - The telescope id - """ - - -class TwoPassStatisticsCalculator(CalibrationCalculator): - """ - Component to calculate statistics from calibration events. - """ - - faulty_pixels_threshold = Float( - 0.1, - help="Percentage of faulty pixels over the camera to conduct second pass with refined shift of the chunk", - ).tag(config=True) - chunk_shift = Int( - 100, - help="Number of samples to shift the extraction chunk for the calculation of the statistical values", - ).tag(config=True) - - def __call__( - self, - input_url, - tel_id, - col_name="image", - ): - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - # Get the extractor - extractor = self.stats_extractor[self.stats_extractor_type.tel[tel_id]] - - # First pass through the whole provided dl1 data - stats_list_firstpass = extractor(dl1_table=dl1_table[tel_id], col_name=col_name) - - # Second pass - stats_list = [] - faultless_previous_chunk = False - for chunk_nr, stats in enumerate(stats_list_firstpass): - # Append faultless stats from the previous chunk - if faultless_previous_chunk: - stats_list.append(stats_list_firstpass[chunk_nr - 1]) - - # Detect faulty pixels over all gain channels - outlier_mask = np.logical_or( - stats.median_outliers[0], stats.std_outliers[0] - ) - if len(stats.median_outliers) == 2: - outlier_mask = np.logical_or( - outlier_mask, - np.logical_or(stats.median_outliers[1], stats.std_outliers[1]), - ) - # Detect faulty chunks by calculating the fraction of faulty pixels over the camera and checking if the threshold is exceeded. - if ( - np.count_nonzero(outlier_mask) / len(outlier_mask) - > self.faulty_pixels_threshold - ): - slice_start, slice_stop = self._get_slice_range( - chunk_nr=chunk_nr, - chunk_size=extractor.chunk_size, - faultless_previous_chunk=faultless_previous_chunk, - last_chunk=len(stats_list_firstpass) - 1, - last_element=len(dl1_table[tel_id]) - 1, - ) - # The two last chunks can be faulty, therefore the length of the sliced table would be smaller than the size of a chunk. - if slice_stop - slice_start > extractor.chunk_size: - # Slice the dl1 table according to the previously calculated start and stop. - dl1_table_sliced = dl1_table[tel_id][slice_start:slice_stop] - # Run the stats extractor on the sliced dl1 table with a chunk_shift - # to remove the period of trouble (carflashes etc.) as effectively as possible. - stats_list_secondpass = extractor( - dl1_table=dl1_table_sliced, chunk_shift=self.chunk_shift - ) - # Extend the final stats list by the stats list of the second pass. - stats_list.extend(stats_list_secondpass) - else: - # Store the last chunk in case the two last chunks are faulty. - stats_list.append(stats_list_firstpass[chunk_nr]) - # Set the boolean to False to track this chunk as faulty for the next iteration. - faultless_previous_chunk = False - else: - # Set the boolean to True to track this chunk as faultless for the next iteration. - faultless_previous_chunk = True - - # Open the output file and store the final stats list. - with open(self.output_path, "wb") as f: - pickle.dump(stats_list, f) - - def _get_slice_range( - self, - chunk_nr, - chunk_size, - faultless_previous_chunk, - last_chunk, - last_element, - ): - slice_start = 0 - if chunk_nr > 0: - slice_start = ( - chunk_size * (chunk_nr - 1) + self.chunk_shift - if faultless_previous_chunk - else chunk_size * chunk_nr + self.chunk_shift - ) - slice_stop = last_element - if chunk_nr < last_chunk: - slice_stop = chunk_size * (chunk_nr + 2) - self.chunk_shift - 1 - - return slice_start, slice_stop - - -class PointingCalculator(CalibrationCalculator): - """ - Component to calculate pointing corrections from interleaved skyfield events. - - Attributes - ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set - telescope_location: dict - The location of the telescope for which the pointing correction is to be calculated - """ - - telescope_location = Dict( - {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147}, - help="Telescope location, longitude and latitude should be expressed in deg, " - "elevation - in meters", - ).tag(config=True) - - min_star_prominence = Integer( - 3, - help="Minimal star prominence over the background in terms of " - "NSB variance std deviations", - ).tag(config=True) - - max_star_magnitude = Float( - 7.0, help="Maximal magnitude of the star to be considered in the " "analysis" - ).tag(config=True) - - psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), - default_value="ComaModel", - help="Name of the PSFModel Subclass to be used.", - ).tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - **kwargs, - ): - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - self.psf = PSFModel.from_name( - self.pas_model_type, subarray=self.subarray, parent=self - ) - - self.location = EarthLocation( - lon=self.telescope_location["longitude"] * u.deg, - lat=self.telescope_location["latitude"] * u.deg, - height=self.telescope_location["elevation"] * u.m, - ) - - def __call__(self, input_url, tel_id): - if self._check_req_data(input_url, tel_id, "flatfield"): - raise KeyError( - "Relative gain not found. Gain calculation needs to be performed first." - ) - - self.tel_id = tel_id - - # first get thecamera geometry and pointing for the file and determine what stars we should see - - with EventSource(input_url, max_events=1) as src: - self.camera_geometry = src.subarray.tel[self.tel_id].camera.geometry - self.focal_length = src.subarray.tel[ - self.tel_id - ].optics.equivalent_focal_length - self.pixel_radius = self.camera_geometry.pixel_width[0] - - event = next(iter(src)) - - self.pointing = SkyCoord( - az=event.pointing.tel[self.telescope_id].azimuth, - alt=event.pointing.tel[self.telescope_id].altitude, - frame="altaz", - obstime=event.trigger.time.utc, - location=self.location, - ) - - stars_in_fov = Vizier.query_region( - self.pointing, radius=Angle(2.0, "deg"), catalog="NOMAD" - )[0] - - stars_in_fov = stars_in_fov[stars_in_fov["Bmag"] < self.max_star_magnitude] - - # Read the whole dl1-like images of pedestal and flat-field data with the TableLoader - input_data = TableLoader(input_url=input_url) - dl1_table = input_data.read_telescope_events_by_id( - telescopes=tel_id, - dl1_images=True, - dl1_parameters=False, - dl1_muons=False, - dl2=False, - simulated=False, - true_images=False, - true_parameters=False, - instrument=False, - pointing=False, - ) - - # get the time and images from the data - - variance_images = dl1_table["variance_image"] - - time = dl1_table["time"] - - # now calibrate the images - - variance_images = self._calibrate_var_images( - self, variance_images, time, input_url - ) - - def _check_req_data(self, url, tel_id, calibration_type): - """ - Check if the prerequisite calibration data exists in the files - - Parameters - ---------- - url : str - URL of file that is to be tested - tel_id : int - The telescope id. - calibration_type : str - Name of the field that is to be looked for e.g. flatfield or - gain - """ - with EventSource(url, max_events=1) as source: - event = next(iter(source)) - - calibration_data = getattr(event.mon.tel[tel_id], calibration_type) - - if calibration_data is None: - return False - - return True - - def _calibrate_var_images(self, var_images, time, calibration_file): - # So i need to use the interpolator classes to read the calibration data - relative_gains = FlatFieldInterpolator( - calibration_file - ) # this assumes gain_file is an hdf5 file. The interpolator will automatically search for the data in the default location when i call the instance - - for i, var_image in enumerate(var_images): - var_images[i].image = np.divide( - var_image.image, - np.square(relative_gains(time[i])), - ) - - return var_images - - class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 933105efeb6..164c7c52210 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -377,12 +377,6 @@ class PointingCalculator(TelescopeComponent): "elevation - in meters", ).tag(config=True) - observed_wavelength = Float( - 0.35, - help="Observed star light wavelength in microns" - "(convolution of blackbody spectrum with camera sensitivity)", - ).tag(config=True) - min_star_prominence = Integer( 3, help="Minimal star prominence over the background in terms of " @@ -403,7 +397,7 @@ class PointingCalculator(TelescopeComponent): ).tag(config=True) psf_model_type = TelescopeParameter( - trait=ComponentName(PSFModel, default_value="ComaModel"), + trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), default_value="ComaModel", help="Name of the PSFModel Subclass to be used.", ).tag(config=True) @@ -429,7 +423,7 @@ def __init__( ): super().__init__( subarray=subarray, - config=config, + stats_extractor="Plain", parent=parent, **kwargs, ) @@ -630,6 +624,7 @@ def _get_accumulated_images(self, data_table): picture_thresh=self.cleaning["pic_thresh"], boundary_thresh=self.cleaning["bound_thresh"], ) + for x in data_table["charge_image"] ] From d1d146c51d29b781c8eeef2b2eb9aaafaf7fe945 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Thu, 12 Sep 2024 11:08:20 +0200 Subject: [PATCH 212/221] Added the fitting --- src/ctapipe/calib/camera/pointing.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 164c7c52210..03f4c7bff20 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd import Vizier # discuss this dependency with max etc. -from astropy.coordinates import Angle, EarthLocation, SkyCoord +from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord from astropy.table import QTable from scipy.odr import ODR, RealData @@ -377,6 +377,12 @@ class PointingCalculator(TelescopeComponent): "elevation - in meters", ).tag(config=True) + observed_wavelength = Float( + 0.35, + help="Observed star light wavelength in microns" + "(convolution of blackbody spectrum with camera sensitivity)", + ).tag(config=True) + min_star_prominence = Integer( 3, help="Minimal star prominence over the background in terms of " @@ -493,6 +499,8 @@ def ingest_data(self, data_table): data_table["variance_images"][0].image ) # get the size of images of the camera we are calibrating + star_labels = [x.label for x in self.stars_in_fov] + # get the accumulated variance images self._get_accumulated_images(data_table) From 7ff48c9504bbbd9e28e038bf781f2fe64cfe9bd7 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Mon, 30 Sep 2024 17:23:17 +0200 Subject: [PATCH 213/221] fixed astroquery import --- src/ctapipe/calib/camera/pointing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 03f4c7bff20..65fffa91200 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -9,9 +9,9 @@ import astropy.units as u import numpy as np import pandas as pd -import Vizier # discuss this dependency with max etc. -from astropy.coordinates import ICRS, AltAz, Angle, EarthLocation, SkyCoord +from astropy.coordinates import Angle, EarthLocation, SkyCoord from astropy.table import QTable +from astroquery.vizier import Vizier # discuss this dependency with max etc. from scipy.odr import ODR, RealData from ctapipe.calib.camera.extractor import StatisticsExtractor From 2d913a951baa615eb582d46de41af978cf2c779d Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 1 Oct 2024 09:15:30 +0200 Subject: [PATCH 214/221] removing merge marks --- src/ctapipe/calib/camera/calibrator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ctapipe/calib/camera/calibrator.py b/src/ctapipe/calib/camera/calibrator.py index cbf5442ed7e..baf3d2f1057 100644 --- a/src/ctapipe/calib/camera/calibrator.py +++ b/src/ctapipe/calib/camera/calibrator.py @@ -21,6 +21,7 @@ __all__ = ["CameraCalibrator"] + @cache def _get_pixel_index(n_pixels): """Cached version of ``np.arange(n_pixels)``""" @@ -45,6 +46,7 @@ def _get_invalid_pixels(n_channels, n_pixels, pixel_status, selected_gain_channe return broken_pixels + class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill From fd5bfc8a126237ba28335a1faedba86805c4d4cf Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 1 Oct 2024 14:21:38 +0200 Subject: [PATCH 215/221] I moved the interpolator classes for flatfield events to to their proper place --- src/ctapipe/calib/camera/extractor.py | 233 --------------- src/ctapipe/calib/camera/pointing.py | 27 +- src/ctapipe/io/interpolation.py | 369 ------------------------ src/ctapipe/monitoring/interpolation.py | 187 ++++++++++++ 4 files changed, 201 insertions(+), 615 deletions(-) delete mode 100644 src/ctapipe/calib/camera/extractor.py delete mode 100644 src/ctapipe/io/interpolation.py diff --git a/src/ctapipe/calib/camera/extractor.py b/src/ctapipe/calib/camera/extractor.py deleted file mode 100644 index 7093d057f20..00000000000 --- a/src/ctapipe/calib/camera/extractor.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -Extraction algorithms to compute the statistics from a sequence of images -""" - -__all__ = [ - "StatisticsExtractor", - "PlainExtractor", - "SigmaClippingExtractor", -] - -from abc import abstractmethod - -import numpy as np -from astropy.stats import sigma_clipped_stats - -from ctapipe.containers import StatisticsContainer -from ctapipe.core import TelescopeComponent -from ctapipe.core.traits import ( - Int, - List, -) - - -class StatisticsExtractor(TelescopeComponent): - sample_size = Int(2500, help="sample size").tag(config=True) - image_median_cut_outliers = List( - [-0.3, 0.3], - help="Interval of accepted image values (fraction with respect to camera median value)", - ).tag(config=True) - image_std_cut_outliers = List( - [-3, 3], - help="Interval (number of std) of accepted image standard deviation around camera median value", - ).tag(config=True) - - def __init__(self, subarray, config=None, parent=None, **kwargs): - """ - Base component to handle the extraction of the statistics - from a sequence of charges and pulse times (images). - - Parameters - ---------- - kwargs - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - - @abstractmethod - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - """ - Call the relevant functions to extract the statistics - for the particular extractor. - - Parameters - ---------- - dl1_table : ndarray - dl1 table with images and times stored in a numpy array of shape - (n_images, n_channels, n_pix). - col_name : string - column name in the dl1 table - - Returns - ------- - List StatisticsContainer: - List of extracted statistics and validity ranges - """ - - -class PlainExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using numpy and scipy functions - """ - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._plain_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _plain_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median over the sample per pixel - pixel_median = np.ma.median(masked_images, axis=0) - - # mean over the sample per pixel - pixel_mean = np.ma.mean(masked_images, axis=0) - - # std over the sample per pixel - pixel_std = np.ma.std(masked_images, axis=0) - - # median of the median over the camera - # median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # outliers from median - image_median_outliers = np.logical_or( - pixel_median < self.image_median_cut_outliers[0], - pixel_median > self.image_median_cut_outliers[1], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - ) - - -class SigmaClippingExtractor(StatisticsExtractor): - """ - Extractor the statistics from a sequence of images - using astropy's sigma clipping functions - """ - - sigma_clipping_max_sigma = Int( - default_value=4, - help="Maximal value for the sigma clipping outlier removal", - ).tag(config=True) - sigma_clipping_iterations = Int( - default_value=5, - help="Number of iterations for the sigma clipping outlier removal", - ).tag(config=True) - - def __call__( - self, dl1_table, masked_pixels_of_sample=None, col_name="image" - ) -> list: - # in python 3.12 itertools.batched can be used - image_chunks = ( - dl1_table[col_name].data[i : i + self.sample_size] - for i in range(0, len(dl1_table[col_name].data), self.sample_size) - ) - time_chunks = ( - dl1_table["time"][i : i + self.sample_size] - for i in range(0, len(dl1_table["time"]), self.sample_size) - ) - - # Calculate the statistics from a sequence of images - stats_list = [] - for images, times in zip(image_chunks, time_chunks): - stats_list.append( - self._sigmaclipping_extraction(images, times, masked_pixels_of_sample) - ) - return stats_list - - def _sigmaclipping_extraction( - self, images, times, masked_pixels_of_sample - ) -> StatisticsContainer: - # ensure numpy array - masked_images = np.ma.array(images, mask=masked_pixels_of_sample) - - # median of the event images - # image_median = np.ma.median(masked_images, axis=-1) - - # mean, median, and std over the sample per pixel - max_sigma = self.sigma_clipping_max_sigma - pixel_mean, pixel_median, pixel_std = sigma_clipped_stats( - masked_images, - sigma=max_sigma, - maxiters=self.sigma_clipping_iterations, - cenfunc="mean", - axis=0, - ) - - # mask pixels without defined statistical values - pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean)) - pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median)) - pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std)) - - unused_values = np.abs(masked_images - pixel_mean) > (max_sigma * pixel_std) - - # only warn for values discard in the sigma clipping, not those from before - # outliers = unused_values & (~masked_images.mask) - - # add outliers identified by sigma clipping for following operations - masked_images.mask |= unused_values - - # median of the median over the camera - median_of_pixel_median = np.ma.median(pixel_median, axis=1) - - # median of the std over the camera - median_of_pixel_std = np.ma.median(pixel_std, axis=1) - - # std of the std over camera - std_of_pixel_std = np.ma.std(pixel_std, axis=1) - - # outliers from median - image_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] - image_median_outliers = np.logical_or( - image_deviation - < self.image_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], - image_deviation - > self.image_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis], - ) - - # outliers from standard deviation - deviation = pixel_std - median_of_pixel_std[:, np.newaxis] - image_std_outliers = np.logical_or( - deviation - < self.image_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], - deviation - > self.image_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis], - ) - - return StatisticsContainer( - validity_start=times[0], - validity_stop=times[-1], - mean=pixel_mean.filled(np.nan), - median=pixel_median.filled(np.nan), - median_outliers=image_median_outliers.filled(True), - std=pixel_std.filled(np.nan), - std_outliers=image_std_outliers.filled(True), - ) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 65fffa91200..cb7f1245de5 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -14,7 +14,6 @@ from astroquery.vizier import Vizier # discuss this dependency with max etc. from scipy.odr import ODR, RealData -from ctapipe.calib.camera.extractor import StatisticsExtractor from ctapipe.containers import StarContainer from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.core import TelescopeComponent @@ -28,7 +27,8 @@ from ctapipe.image import tailcuts_clean from ctapipe.image.psf_model import PSFModel from ctapipe.instrument import CameraGeometry -from ctapipe.io import FlatFieldInterpolator, PointingInterpolator +from ctapipe.monitoring.aggregator import StatisticsAggregator +from ctapipe.monitoring.interpolation import FlatFieldInterpolator, PointingInterpolator __all__ = ["PointingCalculator", "StarImageGenerator"] @@ -359,16 +359,16 @@ class PointingCalculator(TelescopeComponent): Attributes ---------- - stats_extractor: str - The name of the StatisticsExtractor subclass to be used to calculate the statistics of an image set + stats_aggregator: str + The name of the StatisticsAggregator subclass to be used to calculate the statistics of an image set telescope_location: dict The location of the telescope for which the pointing correction is to be calculated """ - stats_extractor = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="Plain"), - default_value="Plain", - help="Name of the StatisticsExtractor Subclass to be used.", + stats_aggregator = TelescopeParameter( + trait=ComponentName(StatisticsAggregator, default_value="PlainAggregator"), + default_value="PlainAggregator", + help="Name of the StatisticsAggregator Subclass to be used.", ).tag(config=True) telescope_location = Dict( @@ -443,8 +443,9 @@ def __init__( lat=self.telescope_location["latitude"] * u.deg, height=self.telescope_location["elevation"] * u.m, ) - self.stats_aggregator = StatisticsExtractor.from_name( - self.stats_extractor, subarray=self.subarray, parent=self + + self.image_aggregator = StatisticsAggregator.from_name( + self.stats_aggregator, subarray=self.subarray, parent=self ) self.set_camera(geometry) @@ -664,14 +665,14 @@ def _get_accumulated_images(self, data_table): np.square(relative_gains(data_table["time"][i]).median), ) - # then turn it into a table that the extractor can read + # then turn it into a table that the aggregator can read variance_image_table = QTable( [data_table["time"], variance_images], names=["time", "image"] ) - # get the cumulative variance images using the statistics extractor and return the value + # get the cumulative variance images using the statistics aggregator and return the value - variance_statistics = self.stats_aggregator( + variance_statistics = self.image_aggregator( variance_image_table, col_name="image" ) diff --git a/src/ctapipe/io/interpolation.py b/src/ctapipe/io/interpolation.py deleted file mode 100644 index e0e27470c99..00000000000 --- a/src/ctapipe/io/interpolation.py +++ /dev/null @@ -1,369 +0,0 @@ -from abc import ABCMeta, abstractmethod -from typing import Any - -import astropy.units as u -import numpy as np -import tables -from astropy.time import Time -from scipy.interpolate import interp1d - -from ctapipe.core import Component, traits - -from .astropy_helpers import read_table - - -class ChunkFunction: - - """ - Chunk Interpolator for the gain and pedestals - Interpolates data so that for each time the value from the latest starting - valid chunk is given or the earliest available still valid chunk for any - pixels without valid data. - - Parameters - ---------- - values : None | np.array - Numpy array of the data that is to be interpolated. - The first dimension needs to be an index over time - times : None | np.array - Time values over which data are to be interpolated - need to be sorted and have same length as first dimension of values - """ - - def __init__( - self, - start_times, - end_times, - values, - bounds_error=True, - fill_value="extrapolate", - assume_sorted=True, - copy=False, - ): - self.values = values - self.start_times = start_times - self.end_times = end_times - self.bounds_error = bounds_error - self.fill_value = fill_value - - def __call__(self, point): - if point < self.start_times[0]: - if self.bounds_error: - raise ValueError("below the interpolation range") - - if self.fill_value == "extrapolate": - return self.values[0] - - else: - a = np.empty(self.values[0].shape) - a[:] = np.nan - return a - - elif point > self.end_times[-1]: - if self.bounds_error: - raise ValueError("above the interpolation range") - - if self.fill_value == "extrapolate": - return self.values[-1] - - else: - a = np.empty(self.values[0].shape) - a[:] = np.nan - return a - - else: - i = np.searchsorted( - self.start_times, point, side="left" - ) # Latest valid chunk - j = np.searchsorted( - self.end_times, point, side="left" - ) # Earliest valid chunk - return np.where( - np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] - ) # Give value for latest chunk unless its nan. If nan give earliest chunk value - - -class Interpolator(Component, metaclass=ABCMeta): - """ - Interpolator parent class. - - Parameters - ---------- - h5file : None | tables.File - A open hdf5 file with read access. - """ - - bounds_error = traits.Bool( - default_value=True, - help="If true, raises an exception when trying to extrapolate out of the given table", - ).tag(config=True) - - extrapolate = traits.Bool( - help="If bounds_error is False, this flag will specify whether values outside" - "the available values are filled with nan (False) or extrapolated (True).", - default_value=False, - ).tag(config=True) - - telescope_data_group = None - required_columns = set() - expected_units = {} - - def __init__(self, h5file=None, **kwargs): - super().__init__(**kwargs) - - if h5file is not None and not isinstance(h5file, tables.File): - raise TypeError("h5file must be a tables.File") - self.h5file = h5file - - self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False) - if self.bounds_error: - self.interp_options["bounds_error"] = True - elif self.extrapolate: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = "extrapolate" - else: - self.interp_options["bounds_error"] = False - self.interp_options["fill_value"] = np.nan - - self._interpolators = {} - - @abstractmethod - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - This method reads input tables and creates instances of the needed interpolators - to be added to _interpolators. The first index of _interpolators needs to be - tel_id, the second needs to be the name of the parameter that is to be interpolated - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are always ``time`` as ``Time`` column and - other columns for the data that is to be interpolated - """ - - pass - - def _check_tables(self, input_table): - missing = self.required_columns - set(input_table.colnames) - if len(missing) > 0: - raise ValueError(f"Table is missing required column(s): {missing}") - for col in self.expected_units: - unit = input_table[col].unit - if unit is None: - if self.expected_units[col] is not None: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - elif not self.expected_units[col].is_equivalent(unit): - if self.expected_units[col] is None: - raise ValueError(f"{col} must have units compatible with 'None'") - else: - raise ValueError( - f"{col} must have units compatible with '{self.expected_units[col].name}'" - ) - - def _check_interpolators(self, tel_id): - if tel_id not in self._interpolators: - if self.h5file is not None: - self._read_parameter_table(tel_id) # might need to be removed - else: - raise KeyError(f"No table available for tel_id {tel_id}") - - def _read_parameter_table(self, tel_id): - input_table = read_table( - self.h5file, - f"{self.telescope_data_group}/tel_{tel_id:03d}", - ) - self.add_table(tel_id, input_table) - - -class PointingInterpolator(Interpolator): - """ - Interpolator for pointing and pointing correction data - """ - - telescope_data_group = "/dl0/monitoring/telescope/pointing" - required_columns = frozenset(["time", "azimuth", "altitude"]) - expected_units = {"azimuth": u.rad, "altitude": u.rad} - - def __call__(self, tel_id, time): - """ - Interpolate alt/az for given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the pointing - - Returns - ------- - altitude : astropy.units.Quantity[deg] - interpolated altitude angle - azimuth : astropy.units.Quantity[deg] - interpolated azimuth angle - """ - - self._check_interpolators(tel_id) - - mjd = time.tai.mjd - az = u.Quantity(self._interpolators[tel_id]["az"](mjd), u.rad, copy=False) - alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False) - return alt, az - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column, ``azimuth`` and ``altitude`` - as quantity columns for pointing and pointing correction data. - """ - - self._check_tables(input_table) - - if not isinstance(input_table["time"], Time): - raise TypeError("'time' column of pointing table must be astropy.time.Time") - - input_table = input_table.copy() - input_table.sort("time") - - az = input_table["azimuth"].quantity.to_value(u.rad) - # prepare azimuth for interpolation by "unwrapping": i.e. turning - # [359, 1] into [359, 361]. This assumes that if we get values like - # [359, 1] the telescope moved 2 degrees through 0, not 358 degrees - # the other way around. This should be true for all telescopes given - # the sampling speed of pointing values and their maximum movement speed. - # No telescope can turn more than 180° in 2 seconds. - az = np.unwrap(az) - alt = input_table["altitude"].quantity.to_value(u.rad) - mjd = input_table["time"].tai.mjd - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) - self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) - - -class FlatFieldInterpolator(Interpolator): - """ - Interpolator for flatfield data - """ - - telescope_data_group = "dl1/calibration/gain" # TBD - required_columns = frozenset(["start_time", "end_time", "gain"]) - expected_units = {"gain": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate flatfield data for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - ffield : array [float] - interpolated flatfield data - """ - - self._check_interpolators(tel_id) - - ffield = self._interpolators[tel_id]["gain"](time) - return ffield - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "gain" - for the flatfield data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("start_time") - start_time = input_table["start_time"] - end_time = input_table["end_time"] - gain = input_table["gain"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["gain"] = ChunkFunction( - start_time, end_time, gain, **self.interp_options - ) - - -class PedestalInterpolator(Interpolator): - """ - Interpolator for Pedestal data - """ - - telescope_data_group = "dl1/calibration/pedestal" # TBD - required_columns = frozenset(["start_time", "end_time", "pedestal"]) - expected_units = {"pedestal": u.one} - - def __call__(self, tel_id, time): - """ - Interpolate pedestal or gain for a given time and tel_id. - - Parameters - ---------- - tel_id : int - telescope id - time : astropy.time.Time - time for which to interpolate the calibration data - - Returns - ------- - pedestal : array [float] - interpolated pedestal values - """ - - self._check_interpolators(tel_id) - - pedestal = self._interpolators[tel_id]["pedestal"](time) - return pedestal - - def add_table(self, tel_id, input_table): - """ - Add a table to this interpolator - - Parameters - ---------- - tel_id : int - Telescope id - input_table : astropy.table.Table - Table of pointing values, expected columns - are ``time`` as ``Time`` column and "pedestal" - for the pedestal data - """ - - self._check_tables(input_table) - - input_table = input_table.copy() - input_table.sort("start_time") - start_time = input_table["start_time"] - end_time = input_table["end_time"] - pedestal = input_table["pedestal"] - self._interpolators[tel_id] = {} - self._interpolators[tel_id]["pedestal"] = ChunkFunction( - start_time, end_time, pedestal, **self.interp_options - ) diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py index 658fe5291b4..9358aea8153 100644 --- a/src/ctapipe/monitoring/interpolation.py +++ b/src/ctapipe/monitoring/interpolation.py @@ -11,6 +11,77 @@ from .astropy_helpers import read_table +class ChunkFunction: + + """ + Chunk Interpolator for the gain and pedestals + Interpolates data so that for each time the value from the latest starting + valid chunk is given or the earliest available still valid chunk for any + pixels without valid data. + + Parameters + ---------- + values : None | np.array + Numpy array of the data that is to be interpolated. + The first dimension needs to be an index over time + times : None | np.array + Time values over which data are to be interpolated + need to be sorted and have same length as first dimension of values + """ + + def __init__( + self, + start_times, + end_times, + values, + bounds_error=True, + fill_value="extrapolate", + assume_sorted=True, + copy=False, + ): + self.values = values + self.start_times = start_times + self.end_times = end_times + self.bounds_error = bounds_error + self.fill_value = fill_value + + def __call__(self, point): + if point < self.start_times[0]: + if self.bounds_error: + raise ValueError("below the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[0] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + elif point > self.end_times[-1]: + if self.bounds_error: + raise ValueError("above the interpolation range") + + if self.fill_value == "extrapolate": + return self.values[-1] + + else: + a = np.empty(self.values[0].shape) + a[:] = np.nan + return a + + else: + i = np.searchsorted( + self.start_times, point, side="left" + ) # Latest valid chunk + j = np.searchsorted( + self.end_times, point, side="left" + ) # Earliest valid chunk + return np.where( + np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1] + ) # Give value for latest chunk unless its nan. If nan give earliest chunk value + + class Interpolator(Component, metaclass=ABCMeta): """ Interpolator parent class. @@ -182,3 +253,119 @@ def add_table(self, tel_id, input_table): self._interpolators[tel_id] = {} self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options) self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options) + + +class FlatFieldInterpolator(Interpolator): + """ + Interpolator for flatfield data + """ + + telescope_data_group = "dl1/calibration/gain" # TBD + required_columns = frozenset(["start_time", "end_time", "gain"]) + expected_units = {"gain": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate flatfield data for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + ffield : array [float] + interpolated flatfield data + """ + + self._check_interpolators(tel_id) + + ffield = self._interpolators[tel_id]["gain"](time) + return ffield + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "gain" + for the flatfield data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] + gain = input_table["gain"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["gain"] = ChunkFunction( + start_time, end_time, gain, **self.interp_options + ) + + +class PedestalInterpolator(Interpolator): + """ + Interpolator for Pedestal data + """ + + telescope_data_group = "dl1/calibration/pedestal" # TBD + required_columns = frozenset(["start_time", "end_time", "pedestal"]) + expected_units = {"pedestal": u.one} + + def __call__(self, tel_id, time): + """ + Interpolate pedestal or gain for a given time and tel_id. + + Parameters + ---------- + tel_id : int + telescope id + time : astropy.time.Time + time for which to interpolate the calibration data + + Returns + ------- + pedestal : array [float] + interpolated pedestal values + """ + + self._check_interpolators(tel_id) + + pedestal = self._interpolators[tel_id]["pedestal"](time) + return pedestal + + def add_table(self, tel_id, input_table): + """ + Add a table to this interpolator + + Parameters + ---------- + tel_id : int + Telescope id + input_table : astropy.table.Table + Table of pointing values, expected columns + are ``time`` as ``Time`` column and "pedestal" + for the pedestal data + """ + + self._check_tables(input_table) + + input_table = input_table.copy() + input_table.sort("start_time") + start_time = input_table["start_time"] + end_time = input_table["end_time"] + pedestal = input_table["pedestal"] + self._interpolators[tel_id] = {} + self._interpolators[tel_id]["pedestal"] = ChunkFunction( + start_time, end_time, pedestal, **self.interp_options + ) From de71d9d07165653c8bcb4fa2553e9668c48382e9 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Tue, 1 Oct 2024 16:09:56 +0200 Subject: [PATCH 216/221] I fixed an issue with the PointingCalculator traits --- src/ctapipe/calib/camera/pointing.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index cb7f1245de5..a6a840b6b84 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -23,6 +23,7 @@ Float, Integer, TelescopeParameter, + Unicode, ) from ctapipe.image import tailcuts_clean from ctapipe.image.psf_model import PSFModel @@ -402,10 +403,8 @@ class PointingCalculator(TelescopeComponent): help="Meteorological parameters in [dimensionless, deg C, hPa]", ).tag(config=True) - psf_model_type = TelescopeParameter( - trait=ComponentName(StatisticsExtractor, default_value="ComaModel"), - default_value="ComaModel", - help="Name of the PSFModel Subclass to be used.", + psf_model_type = Unicode( + "ComaModel", help="Name of the PSFModel Subclass to be used." ).tag(config=True) meteo_parameters = Dict( From a59e8a47c190747e2853ce7d88c2ff83fca33e31 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 2 Oct 2024 14:44:33 +0200 Subject: [PATCH 217/221] Adding some parts to the unit tests --- src/ctapipe/calib/camera/pointing.py | 2 +- .../calib/camera/tests/test_pointing.py | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index a6a840b6b84..890f3b546ae 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -31,7 +31,7 @@ from ctapipe.monitoring.aggregator import StatisticsAggregator from ctapipe.monitoring.interpolation import FlatFieldInterpolator, PointingInterpolator -__all__ = ["PointingCalculator", "StarImageGenerator"] +__all__ = ["PointingCalculator", "StarImageGenerator", "StarTracer"] @cache diff --git a/src/ctapipe/calib/camera/tests/test_pointing.py b/src/ctapipe/calib/camera/tests/test_pointing.py index c29014190cb..1540e9ce3b1 100644 --- a/src/ctapipe/calib/camera/tests/test_pointing.py +++ b/src/ctapipe/calib/camera/tests/test_pointing.py @@ -1,3 +1,64 @@ """ Tests for StatisticsExtractor and related functions """ + +import astropy.units as u +from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.time import Time + +from ctapipe.calib.camera.pointing import StarTracer + +# let's prepare a StarTracer to make realistic images +# we need a maximum magnitude +max_magnitude = 2.0 + +# then some time period. let's take the first minute of 2023 + +times = [ + Time("2023-01-01T00:00:" + str(t), format="isot", scale="utc") + for t in range(10, 40, 5) +] + +# then the location of the first LST + +location = {"longitude": 342.108612, "latitude": 28.761389, "elevation": 2147} + +LST = EarthLocation( + lat=location["latitude"] * u.deg, + lon=location["longitude"] * u.deg, + height=location["elevation"] * u.m, +) + +# some weather data + +meteo_parameters = {"relative_humidity": 0.5, "temperature": 10, "pressure": 790} + +# some wavelength + +obswl = 0.42 * u.micron + +# pointing to the crab nebula + +crab = SkyCoord.from_name("crab nebula") + +alt = [] +az = [] + +for t in times: + contemporary_crab = crab.transform_to( + AltAz( + obstime=t, + location=LST, + relative_humidity=meteo_parameters["relative_humidity"], + temperature=meteo_parameters["temperature"] * u.deg_C, + pressure=meteo_parameters["pressure"] * u.hPa, + ) + ) + alt.append(contemporary_crab.alt.to_value()) + az.append(contemporary_crab.az.to_value()) + +# the LST geometry + +# the LST focal length + +st = StarTracer.from_lookup(max_magnitude, az, alt, times, meteo_parameters, obswl) From 583b40836b2eb32f1e180767ac6a4656e0763519 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 2 Oct 2024 17:46:15 +0200 Subject: [PATCH 218/221] Some cleaning up after rebasing --- .../calib/camera/tests/test_extractors.py | 84 ----- src/ctapipe/io/tests/test_interpolator.py | 184 ---------- src/ctapipe/monitoring/calculator.py | 327 ------------------ .../monitoring/tests/test_calculator.py | 144 -------- 4 files changed, 739 deletions(-) delete mode 100644 src/ctapipe/calib/camera/tests/test_extractors.py delete mode 100644 src/ctapipe/io/tests/test_interpolator.py delete mode 100644 src/ctapipe/monitoring/calculator.py delete mode 100644 src/ctapipe/monitoring/tests/test_calculator.py diff --git a/src/ctapipe/calib/camera/tests/test_extractors.py b/src/ctapipe/calib/camera/tests/test_extractors.py deleted file mode 100644 index a83c93fd1c0..00000000000 --- a/src/ctapipe/calib/camera/tests/test_extractors.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Tests for StatisticsExtractor and related functions -""" - -import numpy as np -import pytest -from astropy.table import QTable - -from ctapipe.calib.camera.extractor import PlainExtractor, SigmaClippingExtractor - - -@pytest.fixture(name="test_plainextractor") -def fixture_test_plainextractor(example_subarray): - """test the PlainExtractor""" - return PlainExtractor(subarray=example_subarray, chunk_size=2500) - - -@pytest.fixture(name="test_sigmaclippingextractor") -def fixture_test_sigmaclippingextractor(example_subarray): - """test the SigmaClippingExtractor""" - return SigmaClippingExtractor(subarray=example_subarray, chunk_size=2500) - - -def test_extractors(test_plainextractor, test_sigmaclippingextractor): - """test basic functionality of the StatisticsExtractors""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - pedestal_dl1_data = np.random.normal(2.0, 5.0, size=(5000, 2, 1855)) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - - pedestal_dl1_table = QTable([times, pedestal_dl1_data], names=("time", "image")) - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - - plain_stats_list = test_plainextractor(dl1_table=pedestal_dl1_table) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].mean - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].mean - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[1].median - 2.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[1].median - 77.0) > 1.5) - - assert not np.any(np.abs(plain_stats_list[0].std - 5.0) > 1.5) - assert not np.any(np.abs(sigmaclipping_stats_list[0].std - 10.0) > 1.5) - - -def test_check_outliers(test_sigmaclippingextractor): - """test detection ability of outliers""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_data[:, 0, 120] = 120.0 - flatfield_dl1_data[:, 1, 67] = 120.0 - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table - ) - - # check if outliers where detected correctly - assert sigmaclipping_stats_list[0].median_outliers[0][120] - assert sigmaclipping_stats_list[0].median_outliers[1][67] - assert sigmaclipping_stats_list[1].median_outliers[0][120] - assert sigmaclipping_stats_list[1].median_outliers[1][67] - - -def test_check_chunk_shift(test_sigmaclippingextractor): - """test the chunk shift option and the boundary case for the last chunk""" - - times = np.linspace(60117.911, 60117.9258, num=5000) - flatfield_dl1_data = np.random.normal(77.0, 10.0, size=(5000, 2, 1855)) - # insert outliers - flatfield_dl1_table = QTable([times, flatfield_dl1_data], names=("time", "image")) - sigmaclipping_stats_list = test_sigmaclippingextractor( - dl1_table=flatfield_dl1_table, chunk_shift=2000 - ) - - # check if three chunks are used for the extraction - assert len(sigmaclipping_stats_list) == 3 diff --git a/src/ctapipe/io/tests/test_interpolator.py b/src/ctapipe/io/tests/test_interpolator.py deleted file mode 100644 index 20f5657c1ae..00000000000 --- a/src/ctapipe/io/tests/test_interpolator.py +++ /dev/null @@ -1,184 +0,0 @@ -import astropy.units as u -import numpy as np -import pytest -import tables -from astropy.table import Table -from astropy.time import Time - -from ctapipe.io.interpolation import ( - FlatFieldInterpolator, - PedestalInterpolator, - PointingInterpolator, -) - -t0 = Time("2022-01-01T00:00:00") - - -def test_azimuth_switchover(): - """Test pointing interpolation""" - - table = Table( - { - "time": t0 + [0, 1, 2] * u.s, - "azimuth": [359, 1, 3] * u.deg, - "altitude": [60, 61, 62] * u.deg, - }, - ) - - interpolator = PointingInterpolator() - interpolator.add_table(1, table) - - alt, az = interpolator(tel_id=1, time=t0 + 0.5 * u.s) - assert u.isclose(az, 360 * u.deg) - assert u.isclose(alt, 60.5 * u.deg) - - -def test_invalid_input(): - """Test invalid pointing tables raise nice errors""" - - wrong_time = Table( - { - "time": [1, 2, 3] * u.s, - "azimuth": [1, 2, 3] * u.deg, - "altitude": [1, 2, 3] * u.deg, - } - ) - - interpolator = PointingInterpolator() - with pytest.raises(TypeError, match="astropy.time.Time"): - interpolator.add_table(1, wrong_time) - - wrong_unit = Table( - { - "time": Time(1.7e9 + np.arange(3), format="unix"), - "azimuth": [1, 2, 3] * u.m, - "altitude": [1, 2, 3] * u.deg, - } - ) - with pytest.raises(ValueError, match="compatible with 'rad'"): - interpolator.add_table(1, wrong_unit) - - wrong_unit = Table( - { - "time": Time(1.7e9 + np.arange(3), format="unix"), - "azimuth": [1, 2, 3] * u.deg, - "altitude": [1, 2, 3], - } - ) - with pytest.raises(ValueError, match="compatible with 'rad'"): - interpolator.add_table(1, wrong_unit) - - -def test_hdf5(tmp_path): - """Test writing interpolated data to file""" - from ctapipe.io import write_table - - table = Table( - { - "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, - "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, - "altitude": np.linspace(70.0, 60.0, 6) * u.deg, - }, - ) - - path = tmp_path / "pointing.h5" - write_table(table, path, "/dl0/monitoring/telescope/pointing/tel_001") - with tables.open_file(path) as h5file: - interpolator = PointingInterpolator(h5file) - alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s) - assert u.isclose(alt, 69 * u.deg) - assert u.isclose(az, 1 * u.deg) - - -def test_bounds(): - """Test invalid pointing tables raise nice errors""" - - table_pointing = Table( - { - "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s, - "azimuth": np.linspace(0.0, 10.0, 6) * u.deg, - "altitude": np.linspace(70.0, 60.0, 6) * u.deg, - }, - ) - - table_pedestal = Table( - { - "start_time": np.arange(0.0, 10.1, 2.0), - "end_time": np.arange(0.5, 10.6, 2.0), - "pedestal": np.reshape(np.random.normal(4.0, 1.0, 1850 * 6), (6, 1850)) - * u.Unit(), - }, - ) - - table_flatfield = Table( - { - "start_time": np.arange(0.0, 10.1, 2.0), - "end_time": np.arange(0.5, 10.6, 2.0), - "gain": np.reshape(np.random.normal(1.0, 1.0, 1850 * 6), (6, 1850)) - * u.Unit(), - }, - ) - - interpolator_pointing = PointingInterpolator() - interpolator_pedestal = PedestalInterpolator() - interpolator_flatfield = FlatFieldInterpolator() - interpolator_pointing.add_table(1, table_pointing) - interpolator_pedestal.add_table(1, table_pedestal) - interpolator_flatfield.add_table(1, table_flatfield) - - error_message = "below the interpolation range" - - with pytest.raises(ValueError, match=error_message): - interpolator_pointing(tel_id=1, time=t0 - 0.1 * u.s) - - with pytest.raises(ValueError, match=error_message): - interpolator_pedestal(tel_id=1, time=-0.1) - - with pytest.raises(ValueError, match=error_message): - interpolator_flatfield(tel_id=1, time=-0.1) - - with pytest.raises(ValueError, match="above the interpolation range"): - interpolator_pointing(tel_id=1, time=t0 + 10.2 * u.s) - - alt, az = interpolator_pointing(tel_id=1, time=t0 + 1 * u.s) - assert u.isclose(alt, 69 * u.deg) - assert u.isclose(az, 1 * u.deg) - - pedestal = interpolator_pedestal(tel_id=1, time=1.0) - assert all(pedestal == table_pedestal["pedestal"][0]) - flatfield = interpolator_flatfield(tel_id=1, time=1.0) - assert all(flatfield == table_flatfield["gain"][0]) - with pytest.raises(KeyError): - interpolator_pointing(tel_id=2, time=t0 + 1 * u.s) - with pytest.raises(KeyError): - interpolator_pedestal(tel_id=2, time=1.0) - with pytest.raises(KeyError): - interpolator_flatfield(tel_id=2, time=1.0) - - interpolator_pointing = PointingInterpolator(bounds_error=False) - interpolator_pedestal = PedestalInterpolator(bounds_error=False) - interpolator_flatfield = FlatFieldInterpolator(bounds_error=False) - interpolator_pointing.add_table(1, table_pointing) - interpolator_pedestal.add_table(1, table_pedestal) - interpolator_flatfield.add_table(1, table_flatfield) - - for dt in (-0.1, 10.1) * u.s: - alt, az = interpolator_pointing(tel_id=1, time=t0 + dt) - assert np.isnan(alt.value) - assert np.isnan(az.value) - - assert all(np.isnan(interpolator_pedestal(tel_id=1, time=-0.1))) - assert all(np.isnan(interpolator_flatfield(tel_id=1, time=-0.1))) - - assert all(np.isnan(interpolator_pedestal(tel_id=1, time=20.0))) - assert all(np.isnan(interpolator_flatfield(tel_id=1, time=20.0))) - - interpolator_pointing = PointingInterpolator(bounds_error=False, extrapolate=True) - interpolator_pointing.add_table(1, table_pointing) - alt, az = interpolator_pointing(tel_id=1, time=t0 - 1 * u.s) - assert u.isclose(alt, 71 * u.deg) - assert u.isclose(az, -1 * u.deg) - - alt, az = interpolator_pointing(tel_id=1, time=t0 + 11 * u.s) - assert u.isclose(alt, 59 * u.deg) - assert u.isclose(az, 11 * u.deg) diff --git a/src/ctapipe/monitoring/calculator.py b/src/ctapipe/monitoring/calculator.py deleted file mode 100644 index d95041cf75c..00000000000 --- a/src/ctapipe/monitoring/calculator.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -Definition of the ``StatisticsCalculator`` class, providing all steps needed to -calculate the montoring data for the camera calibration. -""" - -import numpy as np -from astropy.table import Table, vstack - -from ctapipe.core import TelescopeComponent -from ctapipe.core.traits import ( - ComponentName, - Dict, - Float, - Int, - List, - TelescopeParameter, -) -from ctapipe.monitoring.aggregator import StatisticsAggregator -from ctapipe.monitoring.outlier import OutlierDetector - -__all__ = [ - "StatisticsCalculator", -] - - -class StatisticsCalculator(TelescopeComponent): - """ - Component to calculate statistics from calibration events. - - The ``StatisticsCalculator`` is responsible for calculating various statistics from - calibration events, such as pedestal and flat-field data. It aggregates statistics, - detects outliers, and handles faulty data periods. - This class holds two functions to conduct two different passes over the data with and without - overlapping aggregation chunks. The first pass is conducted with non-overlapping chunks, - while overlapping chunks can be set by the ``chunk_shift`` parameter for the second pass. - The second pass over the data is only conducted in regions of trouble with a high percentage - of faulty pixels exceeding the threshold ``faulty_pixels_threshold``. - """ - - stats_aggregator_type = TelescopeParameter( - trait=ComponentName( - StatisticsAggregator, default_value="SigmaClippingAggregator" - ), - default_value="SigmaClippingAggregator", - help="Name of the StatisticsAggregator subclass to be used.", - ).tag(config=True) - - outlier_detector_list = List( - trait=Dict(), - default_value=None, - allow_none=True, - help=( - "List of dicts containing the name of the OutlierDetector subclass to be used, " - "the aggregated statistic value to which the detector should be applied, " - "and the validity range of the detector. " - "E.g. ``[{'apply_to': 'std', 'name': 'RangeOutlierDetector', 'validity_range': [2.0, 8.0]},]``." - ), - ).tag(config=True) - - chunk_shift = Int( - default_value=None, - allow_none=True, - help=( - "Number of samples to shift the aggregation chunk for the calculation " - "of the statistical values. Only used in the second_pass(), since the " - "first_pass() is conducted with non-overlapping chunks (chunk_shift=None)." - ), - ).tag(config=True) - - faulty_pixels_threshold = Float( - default_value=10.0, - allow_none=True, - help=( - "Threshold in percentage of faulty pixels over the camera " - "to identify regions of trouble." - ), - ).tag(config=True) - - def __init__( - self, - subarray, - config=None, - parent=None, - stats_aggregator=None, - **kwargs, - ): - """ - Parameters - ---------- - subarray: ctapipe.instrument.SubarrayDescription - Description of the subarray. Provides information about the - camera which are useful in calibration. Also required for - configuring the TelescopeParameter traitlets. - config: traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - This is mutually exclusive with passing a ``parent``. - parent: ctapipe.core.Component or ctapipe.core.Tool - Parent of this component in the configuration hierarchy, - this is mutually exclusive with passing ``config`` - stats_aggregator: ctapipe.monitoring.aggregator.StatisticsAggregator - The ``StatisticsAggregator`` to use. If None, the default via the - configuration system will be constructed. - """ - super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) - self.subarray = subarray - - # Initialize the instances of StatisticsAggregator - self.stats_aggregator = {} - if stats_aggregator is None: - for _, _, name in self.stats_aggregator_type: - self.stats_aggregator[name] = StatisticsAggregator.from_name( - name, subarray=self.subarray, parent=self - ) - else: - name = stats_aggregator.__class__.__name__ - self.stats_aggregator_type = [("type", "*", name)] - self.stats_aggregator[name] = stats_aggregator - - # Initialize the instances of OutlierDetector - self.outlier_detectors = {} - if self.outlier_detector_list is not None: - for outlier_detector in self.outlier_detector_list: - self.outlier_detectors[ - outlier_detector["apply_to"] - ] = OutlierDetector.from_name( - name=outlier_detector["name"], - validity_range=outlier_detector["validity_range"], - subarray=self.subarray, - parent=self, - ) - - def first_pass( - self, - table, - tel_id, - masked_pixels_of_sample=None, - col_name="image", - ) -> Table: - """ - Calculate the monitoring data for a given set of events with non-overlapping aggregation chunks. - - This method performs the first pass over the provided data table to calculate - various statistics for calibration purposes. The statistics are aggregated with - non-overlapping chunks (``chunk_shift`` set to None), and faulty pixels are detected - using a list of outlier detectors. - - - Parameters - ---------- - table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and - timestamps of shape (n_images, ) - tel_id : int - Telescope ID for which the calibration is being performed - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing - col_name : str - Column name in the table from which the statistics will be aggregated - - Returns - ------- - astropy.table.Table - Table containing the aggregated statistics, their outlier masks, and the validity of the chunks - """ - # Get the aggregator - aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] - # Pass through the whole provided dl1 table - aggregated_stats = aggregator( - table=table, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=None, - ) - # Detect faulty pixels with multiple instances of ``OutlierDetector`` - outlier_mask = np.zeros_like(aggregated_stats["mean"], dtype=bool) - for aggregated_val, outlier_detector in self.outlier_detectors.items(): - outlier_mask = np.logical_or( - outlier_mask, - outlier_detector(aggregated_stats[aggregated_val]), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats["outlier_mask"] = outlier_mask - # Get valid chunks and add them to the aggregated statistics - aggregated_stats["is_valid"] = self._get_valid_chunks(outlier_mask) - return aggregated_stats - - def second_pass( - self, - table, - valid_chunks, - tel_id, - masked_pixels_of_sample=None, - col_name="image", - ) -> Table: - """ - Conduct a second pass over the data to refine the statistics in regions with a high percentage of faulty pixels. - - This method performs a second pass over the data with a refined shift of the chunk in regions where a high percentage - of faulty pixels were detected during the first pass. Note: Multiple first passes of different calibration events are - performed which may lead to different identification of faulty chunks in rare cases. Therefore a joined list of faulty - chunks is recommended to be passed to the second pass(es) if those different passes use the same ``chunk_size``. - - Parameters - ---------- - table : astropy.table.Table - DL1-like table with images of shape (n_images, n_channels, n_pix), event IDs and timestamps of shape (n_images, ). - valid_chunks : ndarray - Boolean array indicating the validity of each chunk from the first pass. - Note: This boolean array can be a ``logical_and`` from multiple first passes of different calibration events. - tel_id : int - Telescope ID for which the calibration is being performed. - masked_pixels_of_sample : ndarray, optional - Boolean array of masked pixels of shape (n_pix, ) that are not available for processing. - col_name : str - Column name in the table from which the statistics will be aggregated. - - Returns - ------- - astropy.table.Table - Table containing the aggregated statistics after the second pass, their outlier masks, and the validity of the chunks. - """ - # Check if the chunk_shift is set for the second pass - if self.chunk_shift is None: - raise ValueError( - "chunk_shift must be set if second pass over the data is requested" - ) - # Check if at least one chunk is faulty - if np.all(valid_chunks): - raise ValueError( - "All chunks are valid. The second pass over the data is redundant." - ) - # Get the aggregator - aggregator = self.stats_aggregator[self.stats_aggregator_type.tel[tel_id]] - # Conduct a second pass over the data - aggregated_stats_secondpass = [] - faulty_chunks_indices = np.where(~valid_chunks)[0] - for index in faulty_chunks_indices: - # Log information of the faulty chunks - self.log.info( - "Faulty chunk detected in the first pass at index '%s'.", index - ) - # Calculate the start of the slice depending on whether the previous chunk was faulty or not - slice_start = ( - aggregator.chunk_size * index - if index - 1 in faulty_chunks_indices - else aggregator.chunk_size * (index - 1) - ) - # Set the start of the slice to the first element of the dl1 table if out of bound - # and add one ``chunk_shift``. - slice_start = max(0, slice_start) + self.chunk_shift - # Set the end of the slice to the last element of the dl1 table if out of bound - # and subtract one ``chunk_shift``. - slice_end = min(len(table) - 1, aggregator.chunk_size * (index + 2)) - ( - self.chunk_shift - 1 - ) - # Slice the dl1 table according to the previously calculated start and end. - table_sliced = table[slice_start:slice_end] - # Run the stats aggregator on the sliced dl1 table with a chunk_shift - # to sample the period of trouble (carflashes etc.) as effectively as possible. - # Checking for the length of the sliced table to be greater than the ``chunk_size`` - # since it can be smaller if the last two chunks are faulty. Note: The two last chunks - # can be overlapping during the first pass, so we simply ignore them if there are faulty. - if len(table_sliced) > aggregator.chunk_size: - aggregated_stats_secondpass.append( - aggregator( - table=table_sliced, - masked_pixels_of_sample=masked_pixels_of_sample, - col_name=col_name, - chunk_shift=self.chunk_shift, - ) - ) - # Stack the aggregated statistics of each faulty chunk - aggregated_stats_secondpass = vstack(aggregated_stats_secondpass) - # Detect faulty pixels with multiple instances of OutlierDetector of the second pass - outlier_mask_secondpass = np.zeros_like( - aggregated_stats_secondpass["mean"], dtype=bool - ) - for ( - aggregated_val, - outlier_detector, - ) in self.outlier_detectors.items(): - outlier_mask_secondpass = np.logical_or( - outlier_mask_secondpass, - outlier_detector(aggregated_stats_secondpass[aggregated_val]), - ) - # Add the outlier mask to the aggregated statistics - aggregated_stats_secondpass["outlier_mask"] = outlier_mask_secondpass - aggregated_stats_secondpass["is_valid"] = self._get_valid_chunks( - outlier_mask_secondpass - ) - return aggregated_stats_secondpass - - def _get_valid_chunks(self, outlier_mask): - """ - Identify valid chunks based on the outlier mask. - - This method processes the outlier mask to determine which chunks of data - are considered valid or faulty. A chunk is marked as faulty if the percentage - of outlier pixels exceeds a predefined threshold ``faulty_pixels_threshold``. - - Parameters - ---------- - outlier_mask : numpy.ndarray - Boolean array indicating outlier pixels. The shape of the array should - match the shape of the aggregated statistics. - - Returns - ------- - numpy.ndarray - Boolean array where each element indicates whether the corresponding - chunk is valid (True) or faulty (False). - """ - # Check if the camera has two gain channels - if outlier_mask.shape[1] == 2: - # Combine the outlier mask of both gain channels - outlier_mask = np.logical_or( - outlier_mask[:, 0, :], - outlier_mask[:, 1, :], - ) - # Calculate the fraction of faulty pixels over the camera - faulty_pixels_percentage = ( - np.count_nonzero(outlier_mask, axis=-1) / np.shape(outlier_mask)[-1] - ) * 100.0 - # Check for valid chunks if the threshold is not exceeded - valid_chunks = faulty_pixels_percentage < self.faulty_pixels_threshold - return valid_chunks diff --git a/src/ctapipe/monitoring/tests/test_calculator.py b/src/ctapipe/monitoring/tests/test_calculator.py deleted file mode 100644 index b64630f2597..00000000000 --- a/src/ctapipe/monitoring/tests/test_calculator.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Tests for CalibrationCalculator and related functions -""" - -import numpy as np -from astropy.table import Table, vstack -from astropy.time import Time -from traitlets.config.loader import Config - -from ctapipe.monitoring.aggregator import PlainAggregator -from ctapipe.monitoring.calculator import StatisticsCalculator - - -def test_statistics_calculator(example_subarray): - """test basic functionality of the StatisticsCalculator""" - - # Create dummy data for testing - n_images = 5050 - times = Time( - np.linspace(60117.911, 60117.9258, num=n_images), scale="tai", format="mjd" - ) - event_ids = np.linspace(35, 725000, num=n_images, dtype=int) - rng = np.random.default_rng(0) - charge_data = rng.normal(77.0, 10.0, size=(n_images, 2, 1855)) - # Create tables - charge_table = Table( - [times, event_ids, charge_data], - names=("time_mono", "event_id", "image"), - ) - # Initialize the aggregator and calculator - chunk_size = 1000 - aggregator = PlainAggregator(subarray=example_subarray, chunk_size=chunk_size) - chunk_shift = 500 - calculator = StatisticsCalculator( - subarray=example_subarray, - stats_aggregator=aggregator, - chunk_shift=chunk_shift, - ) - # Compute the statistical values - stats = calculator.first_pass(table=charge_table, tel_id=1) - # Set all chunks as faulty to aggregate the statistic values with a "global" chunk shift - valid_chunks = np.zeros_like(stats["is_valid"].data, dtype=bool) - # Run the second pass over the data - stats_chunk_shift = calculator.second_pass( - table=charge_table, valid_chunks=valid_chunks, tel_id=1 - ) - # Stack the statistic values from the first and second pass - stats_stacked = vstack([stats, stats_chunk_shift]) - # Sort the stacked aggregated statistic values by starting time - stats_stacked.sort(["time_start"]) - # Check if the calculated statistical values are reasonable - # for a camera with two gain channels - np.testing.assert_allclose(stats[0]["mean"], 77.0, atol=2.5) - np.testing.assert_allclose(stats[1]["median"], 77.0, atol=2.5) - np.testing.assert_allclose(stats[0]["std"], 10.0, atol=2.5) - np.testing.assert_allclose(stats_chunk_shift[0]["mean"], 77.0, atol=2.5) - np.testing.assert_allclose(stats_chunk_shift[1]["median"], 77.0, atol=2.5) - np.testing.assert_allclose(stats_chunk_shift[0]["std"], 10.0, atol=2.5) - # Check if overlapping chunks of the second pass were aggregated - assert stats_chunk_shift is not None - # Check if the number of aggregated chunks is correct - # In the first pass, the number of chunks is equal to the - # number of images divided by the chunk size plus one - # overlapping chunk at the end. - expected_len_firstpass = n_images // chunk_size + 1 - assert len(stats) == expected_len_firstpass - # In the second pass, the number of chunks is equal to the - # number of images divided by the chunk shift minus the - # number of chunks in the first pass, since we set all - # chunks to be faulty. - expected_len_secondpass = (n_images // chunk_shift) - expected_len_firstpass - assert len(stats_chunk_shift) == expected_len_secondpass - # The total number of aggregated chunks is the sum of the - # number of chunks in the first and second pass. - assert len(stats_stacked) == expected_len_firstpass + expected_len_secondpass - - -def test_outlier_detector(example_subarray): - """test the chunk shift option and the boundary case for the last chunk""" - - # Create dummy data for testing - times = Time( - np.linspace(60117.911, 60117.9258, num=5500), scale="tai", format="mjd" - ) - event_ids = np.linspace(35, 725000, num=5500, dtype=int) - rng = np.random.default_rng(0) - ped_data = rng.normal(2.0, 5.0, size=(5500, 2, 1855)) - # Create table - ped_table = Table( - [times, event_ids, ped_data], - names=("time_mono", "event_id", "image"), - ) - # Create configuration - config = Config( - { - "StatisticsCalculator": { - "stats_aggregator_type": [ - ("id", 1, "SigmaClippingAggregator"), - ], - "outlier_detector_list": [ - { - "apply_to": "mean", - "name": "StdOutlierDetector", - "validity_range": [-2.0, 2.0], - }, - { - "apply_to": "median", - "name": "StdOutlierDetector", - "validity_range": [-3.0, 3.0], - }, - { - "apply_to": "std", - "name": "RangeOutlierDetector", - "validity_range": [2.0, 8.0], - }, - ], - "chunk_shift": 500, - "faulty_pixels_threshold": 9.0, - }, - "SigmaClippingAggregator": { - "chunk_size": 1000, - }, - } - ) - # Initialize the calculator from config - calculator = StatisticsCalculator(subarray=example_subarray, config=config) - # Run the first pass over the data - stats_first_pass = calculator.first_pass(table=ped_table, tel_id=1) - # Run the second pass over the data - stats_second_pass = calculator.second_pass( - table=ped_table, valid_chunks=stats_first_pass["is_valid"].data, tel_id=1 - ) - # Stack the statistic values from the first and second pass - stats_stacked = vstack([stats_first_pass, stats_second_pass]) - # Sort the stacked aggregated statistic values by starting time - stats_stacked.sort(["time_start"]) - # Check if overlapping chunks of the second pass were aggregated - assert stats_second_pass is not None - assert len(stats_stacked) > len(stats_second_pass) - # Check if the calculated statistical values are reasonable - # for a camera with two gain channels - np.testing.assert_allclose(stats_stacked[0]["mean"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_stacked[1]["median"], 2.0, atol=2.5) - np.testing.assert_allclose(stats_stacked[0]["std"], 5.0, atol=2.5) From aa4819fb5217ac8b379ed9dfb5c8459fb3b49591 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 2 Oct 2024 17:53:14 +0200 Subject: [PATCH 219/221] Some more cleaning up --- docs/api-reference/monitoring/calculator.rst | 11 ----------- docs/api-reference/monitoring/index.rst | 2 +- docs/changes/2554.feature.rst | 1 - docs/changes/2609.features.rst | 1 - src/ctapipe/io/__init__.py | 1 - src/ctapipe/monitoring/aggregator.py | 6 +++--- src/ctapipe/monitoring/outlier.py | 12 ++++++------ 7 files changed, 10 insertions(+), 24 deletions(-) delete mode 100644 docs/api-reference/monitoring/calculator.rst delete mode 100644 docs/changes/2554.feature.rst delete mode 100644 docs/changes/2609.features.rst diff --git a/docs/api-reference/monitoring/calculator.rst b/docs/api-reference/monitoring/calculator.rst deleted file mode 100644 index 93a1c1ec861..00000000000 --- a/docs/api-reference/monitoring/calculator.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _calibration_calculator: - -********************** -Calibration Calculator -********************** - - -Reference/API -============= - -.. automodapi:: ctapipe.monitoring.calculator diff --git a/docs/api-reference/monitoring/index.rst b/docs/api-reference/monitoring/index.rst index 47535bff466..c38bf7bb3a9 100644 --- a/docs/api-reference/monitoring/index.rst +++ b/docs/api-reference/monitoring/index.rst @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment. -Code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here. +Currently, only code related to :ref:`stats_aggregator` and :ref:`outlier_detector` is implemented here. Submodules diff --git a/docs/changes/2554.feature.rst b/docs/changes/2554.feature.rst deleted file mode 100644 index 2e6a6356b3a..00000000000 --- a/docs/changes/2554.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add API to extract the statistics from a sequence of images. diff --git a/docs/changes/2609.features.rst b/docs/changes/2609.features.rst deleted file mode 100644 index fac55b285f6..00000000000 --- a/docs/changes/2609.features.rst +++ /dev/null @@ -1 +0,0 @@ -Add calibration calculators which aggregates statistics, detects outliers, handles faulty data chunks. diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index afe96f39430..cfb5fc5501e 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -18,7 +18,6 @@ from .datawriter import DATA_MODEL_VERSION, DataWriter - __all__ = [ "HDF5TableWriter", "HDF5TableReader", diff --git a/src/ctapipe/monitoring/aggregator.py b/src/ctapipe/monitoring/aggregator.py index 8102b21e855..ee8b5566c7c 100644 --- a/src/ctapipe/monitoring/aggregator.py +++ b/src/ctapipe/monitoring/aggregator.py @@ -57,7 +57,7 @@ def __call__( and call the relevant function of the particular aggregator to compute aggregated statistic values. The chunks are generated in a way that ensures they do not overflow the bounds of the table. - If ``chunk_shift`` is None, chunks will not overlap, but the last chunk is ensured to be - of size ``chunk_size``, even if it means the last two chunks will overlap. + of size `chunk_size`, even if it means the last two chunks will overlap. - If ``chunk_shift`` is provided, it will determine the number of samples to shift between the start of consecutive chunks resulting in an overlap of chunks. Chunks that overflows the bounds of the table are not considered. @@ -65,8 +65,8 @@ def __call__( Parameters ---------- table : astropy.table.Table - table with images of shape (n_images, n_channels, n_pix), event IDs and - timestamps of shape (n_images, ) + table with images of shape (n_images, n_channels, n_pix) + and timestamps of shape (n_images, ) masked_pixels_of_sample : ndarray, optional boolean array of masked pixels of shape (n_pix, ) that are not available for processing chunk_shift : int, optional diff --git a/src/ctapipe/monitoring/outlier.py b/src/ctapipe/monitoring/outlier.py index b2b84711cde..4ac387bbcd3 100644 --- a/src/ctapipe/monitoring/outlier.py +++ b/src/ctapipe/monitoring/outlier.py @@ -81,7 +81,7 @@ class MedianOutlierDetector(OutlierDetector): the configurable factors and the camera median of the statistic values. """ - validity_range = List( + median_range_factors = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -98,8 +98,8 @@ def __call__(self, column): # Detect outliers based on the deviation of the median distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.validity_range[0] * camera_median[:, :, np.newaxis], - deviation > self.validity_range[1] * camera_median[:, :, np.newaxis], + deviation < self.median_range_factors[0] * camera_median[:, :, np.newaxis], + deviation > self.median_range_factors[1] * camera_median[:, :, np.newaxis], ) return outliers @@ -112,7 +112,7 @@ class StdOutlierDetector(OutlierDetector): the configurable factors and the camera standard deviation of the statistic values. """ - validity_range = List( + std_range_factors = List( trait=Float(), default_value=[-1.0, 1.0], help=( @@ -131,7 +131,7 @@ def __call__(self, column): # Detect outliers based on the deviation of the standard deviation distribution deviation = column - camera_median[:, :, np.newaxis] outliers = np.logical_or( - deviation < self.validity_range[0] * camera_std[:, :, np.newaxis], - deviation > self.validity_range[1] * camera_std[:, :, np.newaxis], + deviation < self.std_range_factors[0] * camera_std[:, :, np.newaxis], + deviation > self.std_range_factors[1] * camera_std[:, :, np.newaxis], ) return outliers From 666e1d1b12c261c27190425a996d2ea3111ec9d9 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Wed, 2 Oct 2024 17:55:12 +0200 Subject: [PATCH 220/221] last cleanup --- src/ctapipe/monitoring/tests/test_outlier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/monitoring/tests/test_outlier.py b/src/ctapipe/monitoring/tests/test_outlier.py index 61f1d8cb91d..da7d7619b33 100644 --- a/src/ctapipe/monitoring/tests/test_outlier.py +++ b/src/ctapipe/monitoring/tests/test_outlier.py @@ -56,7 +56,7 @@ def test_median_detection(example_subarray): # In this test, the interval [-0.9, 8] corresponds to multiplication factors # typical used for the median values of charge images of flat-field events detector = MedianOutlierDetector( - subarray=example_subarray, validity_range=[-0.9, 8.0] + subarray=example_subarray, median_range_factors=[-0.9, 8.0] ) # Detect outliers outliers = detector(table["median"]) @@ -89,7 +89,7 @@ def test_std_detection(example_subarray): # typical used for the std values of charge images of flat-field events # and median (and std) values of charge images of pedestal events detector = StdOutlierDetector( - subarray=example_subarray, validity_range=[-15.0, 15.0] + subarray=example_subarray, std_range_factors=[-15.0, 15.0] ) ff_outliers = detector(ff_table["std"]) ped_outliers = detector(ped_table["median"]) From 885bfcf1e1d55c7bf66abc59dc3617d06b5aa1a8 Mon Sep 17 00:00:00 2001 From: Christoph Toennis Date: Fri, 4 Oct 2024 16:28:40 +0200 Subject: [PATCH 221/221] Switching to only test fitting, not cleaning --- src/ctapipe/calib/camera/pointing.py | 107 ++++++++++++------ .../calib/camera/tests/test_pointing.py | 27 ++++- src/ctapipe/containers.py | 3 +- src/ctapipe/monitoring/interpolation.py | 3 +- 4 files changed, 95 insertions(+), 45 deletions(-) diff --git a/src/ctapipe/calib/camera/pointing.py b/src/ctapipe/calib/camera/pointing.py index 52acea1204a..283ee919a69 100644 --- a/src/ctapipe/calib/camera/pointing.py +++ b/src/ctapipe/calib/camera/pointing.py @@ -146,7 +146,7 @@ def StarImageGenerator( :param dict psf_model_pars: psf model parameters """ camera_geometry = CameraGeometry.from_name(camera_name) - psf = PSFModel.from_name(self.psf_model_type, subarray=self.subarray, parent=self) + psf = PSFModel.from_name(self.psf_model_type) psf.update_model_parameters(psf_model_pars) image = np.zeros(len(camera_geometry)) for r, p, m in zip(radius, phi, magnitude): @@ -353,6 +353,13 @@ def get_expected_star_pixels(self, t, focal_correction=0.0): return res + def get_star_mask(self, t, focal_correction=0.0): + mask = np.full(len(self.camera_geometry), True) + pixels = self.get_expected_star_pixels(t, focal_correction=focal_correction) + mask[pixels] = False + + return mask + class PointingCalculator(TelescopeComponent): """ @@ -418,6 +425,10 @@ class PointingCalculator(TelescopeComponent): focal_length = Float(1.0, help="Camera focal length in meters").tag(config=True) + camera_name = Unicode("LSTCam", help="Name of the camera to be calibrated").tag( + config=True + ) + def __init__( self, subarray, @@ -447,7 +458,7 @@ def __init__( self.stats_aggregator, subarray=self.subarray, parent=self ) - self.set_camera(geometry) + self.set_camera(self.camera_name) def set_camera(self, geometry, focal_lengh): if isinstance(geometry, str): @@ -477,24 +488,6 @@ def ingest_data(self, data_table): self.broken_pixels = np.unique(np.where(data_table["unusable_pixels"])) - for azimuth, altitude, time in zip( - data_table["telescope_pointing_azimuth"], - data_table["telescope_pointing_altitude"], - data_table["time"], - ): - _pointing = SkyCoord( - az=azimuth, - alt=altitude, - frame="altaz", - obstime=time, - location=self.location, - obswl=self.observed_wavelength * u.micron, - relative_humidity=self.meteo_parameters["relative_humidity"], - temperature=self.meteo_parameters["temperature"] * u.deg_C, - pressure=self.meteo_parameters["pressure"] * u.hPa, - ) - self.pointing.append(_pointing) - self.image_size = len( data_table["variance_images"][0].image ) # get the size of images of the camera we are calibrating @@ -635,11 +628,9 @@ def _get_accumulated_images(self, data_table): shower_mask = copy.deepcopy(light_mask) - star_pixels = [ - self.tracer.get_expected_star_pixels(t) for t in data_table["time"] - ] + star_mask = [self.tracer.get_star_mask(t) for t in data_table["time"]] - light_mask[:, star_pixels] = True + light_mask[~star_mask] = True if self.broken_pixels is not None: light_mask[:, self.broken_pixels] = True @@ -672,19 +663,11 @@ def _get_accumulated_images(self, data_table): variance_image_table, col_name="image" ) - self.accumulated_times = np.array( - [x.validity_start for x in variance_statistics] - ) - - accumulated_images = np.array([x.mean for x in variance_statistics]) + self.accumulated_times = variance_statistics["time_start"] - star_pixels = [ - self.tracer.get_expected_star_pixels(t) for t in data_table["time"] - ] + accumulated_images = variance_statistics["mean"] - star_mask = np.ones(self.image_size, dtype=bool) - - star_mask[star_pixels] = False + star_mask = [self.tracer.get_star_mask(t) for t in data_table["time"]] # get NSB values @@ -693,6 +676,56 @@ def _get_accumulated_images(self, data_table): self.clean_images = np.array([x - y for x, y in zip(accumulated_images, nsb)]) + def make_mispointed_data(self, p, pointing_table): + self.tracer = StarTracer.from_lookup( + pointing_table["telescope_pointing_azimuth"], + pointing_table["telescope_pointing_altitude"], + pointing_table["time"], + self.meteo_parameters, + self.observed_wavelength, + self.camera_geometry, + self.focal_length, + self.telescope_location, + ) + + stars = self.tracer.get_star_labels() + + self.accumulated_times = pointing_table["time"] + + self.all_containers = [] + + for t in pointing_table["time"]: + self.all_containers.append([]) + + for star in stars: + x, y = self.tracer.get_position_in_camera(t, star) + r, phi = cart2pol(x, y) + x_mispoint, y_mispoint = self.tracer.get_position_in_camera( + t, star, offset=p + ) + r_mispoint, phi_mispoint = cart2pol(x_mispoint, y_mispoint) + container = StarContainer( + label=star, + magnitude=self.tracer.get_magnitude(star), + expected_x=x, + expected_y=y, + expected_r=r * u.m, + expected_phi=phi * u.rad, + timestamp=t, + reco_x=x_mispoint, + reco_y=y_mispoint, + reco_r=r_mispoint * u.m, + reco_phi=phi_mispoint * u.rad, + reco_dx=0.05 * u.m, + reco_dy=0.05 * u.m, + reco_dr=0.05 * u.m, + reco_dphi=0.15 * u.rad, + ) + + self.all_containers[-1].append(container) + + return self.all_containers + def fit_function(self, p, t): """ Construct the fit function for the pointing correction @@ -715,7 +748,7 @@ def fit_function(self, p, t): return coord_list - def fit(self): + def fit(self, init_mispointing=(0, 0)): """ Performs the ODR fit of stars trajectories and saves the results as self.fit_results @@ -734,7 +767,7 @@ def fit(self): errors.append(star.reco_dx) errors.append(star.reco_dy) - rdata = RealData(x=[t], y=data, sy=self.errors) + rdata = RealData(x=[t], y=data, sy=errors) odr = ODR(rdata, self.fit_function, beta0=init_mispointing) fit_summary = odr.run() fit_results = pd.DataFrame( diff --git a/src/ctapipe/calib/camera/tests/test_pointing.py b/src/ctapipe/calib/camera/tests/test_pointing.py index 1540e9ce3b1..22b1f446f6b 100644 --- a/src/ctapipe/calib/camera/tests/test_pointing.py +++ b/src/ctapipe/calib/camera/tests/test_pointing.py @@ -4,9 +4,11 @@ import astropy.units as u from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.table import QTable from astropy.time import Time -from ctapipe.calib.camera.pointing import StarTracer +from ctapipe.calib.camera.pointing import PointingCalculator +from ctapipe.instrument.camera.geometry import CameraGeometry # let's prepare a StarTracer to make realistic images # we need a maximum magnitude @@ -35,7 +37,7 @@ # some wavelength -obswl = 0.42 * u.micron +obswl = 0.35 * u.micron # pointing to the crab nebula @@ -44,6 +46,8 @@ alt = [] az = [] +# get the local pointing values + for t in times: contemporary_crab = crab.transform_to( AltAz( @@ -57,8 +61,23 @@ alt.append(contemporary_crab.alt.to_value()) az.append(contemporary_crab.az.to_value()) +# next i make the pointing table for the fake data generator + +pointing_table = QTable( + [alt, az, times], + names=["telescope_pointing_altitude", "telescope_pointing_azimuth", "time"], +) + # the LST geometry -# the LST focal length +geometry = CameraGeometry.from_name(name="LSTCam") + +# now set up the PointingCalculator + +pc = PointingCalculator(geometry) + +# now make fake mispointed data + +pc.make_mispointed_data((1.0, -1.0), pointing_table) -st = StarTracer.from_lookup(max_magnitude, az, alt, times, meteo_parameters, obswl) +pc.fit() diff --git a/src/ctapipe/containers.py b/src/ctapipe/containers.py index a706d50627d..ad4e3199a73 100644 --- a/src/ctapipe/containers.py +++ b/src/ctapipe/containers.py @@ -425,8 +425,7 @@ class MorphologyContainer(Container): class StatisticsContainer(Container): """Store descriptive statistics of a chunk of images""" - extraction_start = Field(np.float32(nan), "start of the extraction chunk") - extraction_stop = Field(np.float32(nan), "stop of the extraction chunk") + n_events = Field(-1, "number of events used for the extraction of the statistics") mean = Field( None, "mean of a pixel-wise quantity for each channel" diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py index fda145a12a7..7e216d487b7 100644 --- a/src/ctapipe/monitoring/interpolation.py +++ b/src/ctapipe/monitoring/interpolation.py @@ -9,8 +9,6 @@ from ctapipe.core import Component, traits -from .astropy_helpers import read_table - class ChunkFunction: @@ -175,6 +173,7 @@ def _check_interpolators(self, tel_id): def _read_parameter_table(self, tel_id): # prevent circular import between io and monitoring + from ..io import read_table input_table = read_table( self.h5file,