Merge pull request #280 from cta-observatory/dl2_production

Look for the closest declination line and adequate NSB random forest model for DL2 production

Author: marialainez

Diff for: environment.yml

@@ -21,10 +21,10 @@ dependencies:
   - click
   - pymongo
   - scikit-learn=1.2
-  - ctapipe_io_lst=0.24
+  - ctapipe_io_lst >=0.25.1,<0.26a0
   - ctaplot~=0.6.4
   - pyirf~=0.10
-  - lstchain>=0.10.13
+  - lstchain>=0.10.15
   - tenacity
   - pip:
       - lstcam_calib

Diff for: extra/history_files/sequence_LST1_04185.0001.history

@@ -0,0 +1,3 @@
+04185.0001 lstchain_data_r0_to_dl1 v0.7.0 Thu Mar 25 12:53:40 UTC 2021 calibration.Run04183.0000.hdf5 drs4_pedestal.Run04179.0000.fits 0
+04185.0001 lstchain_dl1ab tailcut105 Thu Mar 25 13:06:09 UTC 2021 dl1_LST-1.Run04185.0001.h5 lstchain_standard_config_tailcut84.json 0
+04185.0001 lstchain_check_dl1 tailcut105 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0001.h5 None 0

Diff for: extra/history_files/sequence_LST1_04185.0010.history

@@ -1,4 +1,4 @@
 04185.0010 lstchain_data_r0_to_dl1 v0.7.0 Thu Mar 25 12:53:40 UTC 2021 calibration.Run04183.0000.hdf5 drs4_pedestal.Run04179.0000.fits 0
 04185.0010 lstchain_dl1ab tailcut84 Thu Mar 25 13:06:09 UTC 2021 dl1_LST-1.Run04185.0010.h5 lstchain_standard_config_tailcut84.json 0
 04185.0010 lstchain_check_dl1 tailcut84 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
-04185.0010 lstchain_dl1_to_dl2 model1 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
+04185.0010 lstchain_dl1_to_dl2 tailcut84/nsb_tuning_0.14 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0

Diff for: pyproject.toml

@@ -33,7 +33,7 @@ classifiers = [
 requires-python = ">=3.10"
 dependencies = [
     "astropy~=5.0",
-    "lstchain>=0.10.13",
+    "lstchain>=0.10.15",
     "matplotlib",
     "numpy",
     "pandas",
@@ -76,6 +76,7 @@ reprocess_longterm = "osa.scripts.reprocess_longterm:main"
 gain_selection = "osa.scripts.gain_selection:main"
 update_source_catalog = "osa.scripts.update_source_catalog:main"
 gainsel_webmaker = "osa.scripts.gainsel_webmaker:main"
+sequencer_catB_tailcuts = "osa.scripts.sequencer_catB_tailcuts:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

Diff for: src/osa/configs/options.py

@@ -15,6 +15,8 @@
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None

Diff for: src/osa/configs/sequencer.cfg

@@ -14,14 +14,18 @@ DRIVE_DIR: %(MONITORING)s/DrivePositioning
 RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
+TAILCUTS_FINDER_DIR: %(BASE)s/auxiliary/TailCuts
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
@@ -45,22 +49,26 @@ DL2_PROD_ID: model2
 #charge_calibration: onsite_create_calibration_file
 drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file
 charge_calibration: lstcam_calib_onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
 r0_to_dl1: lstchain_data_r0_to_dl1
 dl1ab: lstchain_dl1ab
 check_dl1: lstchain_check_dl1
 dl1_to_dl2: lstchain_dl1_to_dl2
+tailcuts_finder: lstchain_find_tailcuts
 
 # To be set by the user
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: False
+apply_standard_dl1b_config: False
 use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
-rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
+mc_prod: 20240918_v0.10.12_allsky_
 dl3_config: /software/lstchain/data/dl3_std_config.json
 max_tries: 3
 

Diff for: src/osa/conftest.py

@@ -31,8 +31,8 @@
 date = datetime.fromisoformat("2020-01-17")
 nightdir = date_to_dir(date)
 prod_id = "v0.1.0"
-dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
-dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
+dl1_prod_id = "tailcut84"
+dl2_prod_id = "tailcut84/nsb_tuning_0.14"
 
 
 @pytest.fixture(scope="session")
@@ -412,6 +412,9 @@ def sequence_list(
     r0_data,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Creates a sequence list from a run summary file."""
     options.directory = running_analysis_dir
@@ -574,10 +577,131 @@ def database(osa_dir):
 
 
 @pytest.fixture(scope="session")
-def gain_selection_flag_file(osa_dir):
+def rf_models_allsky_basedir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_model_path(rf_models_allsky_basedir):
+    mc_prod = "20240131_allsky_v0.10.5_all_dec_base"
+    declination_str = "dec_2276"
+    rf_model_path = rf_models_allsky_basedir / mc_prod / declination_str
+    rf_model_path.mkdir(parents=True, exist_ok=True)
+    return rf_model_path
 
+  
+@pytest.fixture(scope="session")
+def gain_selection_flag_file(osa_dir):
     GainSel_dir = osa_dir / "GainSel" / "20200117"
     GainSel_dir.mkdir(parents=True, exist_ok=True)
     file = GainSel_dir / "GainSelFinished.txt"
     file.touch()
     return file
+
+
+@pytest.fixture(scope="session")
+def catB_closed_file(running_analysis_dir):
+
+    catB_closed_file = running_analysis_dir / "catB_00003.closed"
+    catB_closed_file.touch()
+    return catB_closed_file
+
+
+@pytest.fixture(scope="session")
+def catB_calib_base_dir(monitoring_dir):
+
+    catB_calib_base_dir = monitoring_dir / "PixelCalibration" / "Cat-B"
+    catB_calib_base_dir.mkdir(parents=True, exist_ok=True)
+    return catB_calib_base_dir
+
+
+@pytest.fixture(scope="session")
+def catB_calibration_file(catB_calib_dir):
+
+    catB_calib_dir = catB_calib_base_dir / "calibration" / nightdir / prod_id
+    catB_calib_file = catB_calib_dir / "cat_B_calibration_filters_52.Run00003.h5"
+    catB_calib_file.touch()
+    return catB_calib_file
+
+
+@pytest.fixture(scope="session")
+def tailcuts_finder_dir(base_test_dir):
+    tailcuts_finder_dir = base_test_dir / "auxiliary" / "TailCuts"
+    tailcuts_finder_dir.mkdir(parents=True, exist_ok=True)
+    return tailcuts_finder_dir
+
+
+@pytest.fixture(scope="session")
+def dl1b_config_files(tailcuts_finder_dir):
+    config_information = dedent(
+        """\
+            {
+            "tailcuts_clean_with_pedestal_threshold": {
+                "picture_thresh": 8,
+                "boundary_thresh": 4,
+                "sigma": 2.5,
+                "keep_isolated_pixels": false,
+                "min_number_picture_neighbors": 2,
+                "use_only_main_island": false,
+                "delta_time": 2
+            },
+            "dynamic_cleaning": {
+                "apply": true,
+                "threshold": 267,
+                "fraction_cleaning_intensity": 0.03
+            }
+        }"""
+    )
+    config_file1 = tailcuts_finder_dir / "dl1ab_Run01807.json"
+    config_file1.touch()
+    config_file1.write_text(config_information)
+    config_file2 = tailcuts_finder_dir / "dl1ab_Run01808.json"
+    config_file2.touch()
+    config_file2.write_text(config_information)
+    config_file3 = tailcuts_finder_dir / "dl1ab_Run04185.json"
+    config_file3.touch()
+    config_file3.write_text(config_information)
+    return config_file1, config_file2, config_file3
+
+
+@pytest.fixture(scope="session")
+def tailcuts_log_files(tailcuts_finder_dir):
+    log_information = dedent(
+        """\
+    Median of 95% quantile of pedestal charge: 5.416 p.e.
+
+    Additional NSB rate (over dark MC): 0.2221 p.e./ns
+    lstchain_find_tailcuts finished successfully!
+    """
+    )
+    log_file1 = tailcuts_finder_dir / "log_find_tailcuts_Run01807.log"
+    log_file1.touch()
+    log_file1.write_text(log_information)
+    log_file2 = tailcuts_finder_dir / "log_find_tailcuts_Run01808.log"
+    log_file2.touch()
+    log_file2.write_text(log_information)
+    log_file3 = tailcuts_finder_dir / "log_find_tailcuts_Run04185.log"
+    log_file3.touch()
+    log_file3.write_text(log_information)
+    return log_file1, log_file2, log_file3
+
+
+@pytest.fixture(scope="session")
+def rf_models_base_dir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_models(rf_models_base_dir):
+    rf_models_prefix = cfg.get("lstchain", "mc_prod")
+    rf_models_path1 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.00"
+    rf_models_path1.mkdir(parents=True, exist_ok=True)
+    rf_models_path2 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_2276"
+    rf_models_path2.mkdir(parents=True, exist_ok=True)
+    rf_models_path3 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_4822"
+    rf_models_path3.mkdir(parents=True, exist_ok=True)
+    return rf_models_path1, rf_models_path2, rf_models_path3

Diff for: src/osa/high_level/significance.py

@@ -26,7 +26,8 @@
 from osa.configs.config import cfg
 from osa.nightsummary.extract import get_source_list
 from osa.paths import DEFAULT_CFG, destination_dir, analysis_path
-from osa.utils.cliopts import get_prod_id, get_dl2_prod_id
+from osa.utils.cliopts import get_prod_id
+from osa.paths import get_dl2_prod_id
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, YESTERDAY
 
@@ -209,7 +210,6 @@ def main(
     flat_date = date_to_dir(date)
     options.tel_id = telescope
     options.prod_id = get_prod_id()
-    options.dl2_prod_id = get_dl2_prod_id()
     options.directory = analysis_path(options.tel_id)
     dl2_directory = Path(cfg.get("LST1", "DL2_DIR"))
     highlevel_directory = destination_dir("HIGH_LEVEL", create_dir=True)
@@ -232,7 +232,7 @@ def main(
                 dl2_directory
                 / flat_date
                 / options.prod_id
-                / options.dl2_prod_id
+                / get_dl2_prod_id(run)
                 / f"dl2_LST-1.Run{run:05d}.h5"
             )
             df = pd.concat([df, pd.read_hdf(input_file, key=dl2_params_lstcam_key)])

Diff for: src/osa/high_level/tests/test_significance.py

@@ -10,6 +10,9 @@ def test_significance(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     output = sp.run(
         ["theta2_significance", "-d", "2020-01-17", "-s", "LST1"],