add test for get_RF_model function + adapt tests

Author: marialainez

Diff for: extra/history_files/sequence_LST1_04185.0010.history

@@ -1,4 +1,4 @@
 04185.0010 lstchain_data_r0_to_dl1 v0.7.0 Thu Mar 25 12:53:40 UTC 2021 calibration.Run04183.0000.hdf5 drs4_pedestal.Run04179.0000.fits 0
 04185.0010 lstchain_dl1ab tailcut84 Thu Mar 25 13:06:09 UTC 2021 dl1_LST-1.Run04185.0010.h5 lstchain_standard_config_tailcut84.json 0
 04185.0010 lstchain_check_dl1 tailcut84 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
-04185.0010 lstchain_dl1_to_dl2 model1 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
+04185.0010 lstchain_dl1_to_dl2 tailcut84/nsb_tuning_0.14 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0

Diff for: src/osa/configs/sequencer.cfg

@@ -62,12 +62,12 @@ tailcuts_finder: lstchain_find_tailcuts
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
-apply_catB_calibration: True
+apply_catB_calibration: False
 apply_standard_dl1b_config: False
 use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
-mc_prod: 20240918_v0.10.12
+mc_prod: 20240918_v0.10.12_allsky_
 dl3_config: /software/lstchain/data/dl3_std_config.json
 max_tries: 3
 

Diff for: src/osa/conftest.py

@@ -412,6 +412,9 @@ def sequence_list(
     r0_data,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Creates a sequence list from a run summary file."""
     options.directory = running_analysis_dir
@@ -624,7 +627,7 @@ def catB_calibration_file(catB_calib_dir):
 
 
 @pytest.fixture(scope="session")
-def dl1b_config_file(running_analysis_dir):
+def dl1b_config_files(running_analysis_dir):
     config_information = dedent(
         """\
             {
@@ -644,8 +647,54 @@ def dl1b_config_file(running_analysis_dir):
             }
         }"""
     )
-    config_file = running_analysis_dir / "dl1ab_Run00003.json"
-    config_file.touch()
-    config_file.write_text(config_information)
-    return config_file
+    config_file1 = running_analysis_dir / "dl1ab_Run01807.json"
+    config_file1.touch()
+    config_file1.write_text(config_information)
+    config_file2 = running_analysis_dir / "dl1ab_Run01808.json"
+    config_file2.touch()
+    config_file2.write_text(config_information)
+    config_file3 = running_analysis_dir / "dl1ab_Run04185.json"
+    config_file3.touch()
+    config_file3.write_text(config_information)
+    return config_file1, config_file2, config_file3
 
+
+@pytest.fixture(scope="session")
+def tailcuts_log_files(running_analysis_dir):
+    log_information = dedent(
+        """\
+    Median of 95% quantile of pedestal charge: 5.416 p.e.
+
+    Additional NSB rate (over dark MC): 0.2221 p.e./ns
+    lstchain_find_tailcuts finished successfully!
+    """
+    )
+    log_file1 = running_analysis_dir / "log_find_tailcuts_Run01807.log"
+    log_file1.touch()
+    log_file1.write_text(log_information)
+    log_file2 = running_analysis_dir / "log_find_tailcuts_Run01808.log"
+    log_file2.touch()
+    log_file2.write_text(log_information)
+    log_file3 = running_analysis_dir / "log_find_tailcuts_Run04185.log"
+    log_file3.touch()
+    log_file3.write_text(log_information)
+    return log_file1, log_file2, log_file3
+
+
+@pytest.fixture(scope="session")
+def rf_models_base_dir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_models(rf_models_base_dir):
+    rf_models_prefix = cfg.get("lstchain", "mc_prod")
+    rf_models_path1 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.00"
+    rf_models_path1.mkdir(parents=True, exist_ok=True)
+    rf_models_path2 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_2276"
+    rf_models_path2.mkdir(parents=True, exist_ok=True)
+    rf_models_path3 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_4822"
+    rf_models_path3.mkdir(parents=True, exist_ok=True)
+    return rf_models_path1, rf_models_path2, rf_models_path3

Diff for: src/osa/high_level/significance.py

@@ -26,7 +26,8 @@
 from osa.configs.config import cfg
 from osa.nightsummary.extract import get_source_list
 from osa.paths import DEFAULT_CFG, destination_dir, analysis_path
-from osa.utils.cliopts import get_prod_id, get_dl2_prod_id
+from osa.utils.cliopts import get_prod_id
+from osa.paths import get_dl2_prod_id
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, YESTERDAY
 
@@ -209,7 +210,6 @@ def main(
     flat_date = date_to_dir(date)
     options.tel_id = telescope
     options.prod_id = get_prod_id()
-    options.dl2_prod_id = get_dl2_prod_id()
     options.directory = analysis_path(options.tel_id)
     dl2_directory = Path(cfg.get("LST1", "DL2_DIR"))
     highlevel_directory = destination_dir("HIGH_LEVEL", create_dir=True)
@@ -232,7 +232,7 @@ def main(
                 dl2_directory
                 / flat_date
                 / options.prod_id
-                / options.dl2_prod_id
+                / get_dl2_prod_id(run)
                 / f"dl2_LST-1.Run{run:05d}.h5"
             )
             df = pd.concat([df, pd.read_hdf(input_file, key=dl2_params_lstcam_key)])

Diff for: src/osa/high_level/tests/test_significance.py

@@ -10,6 +10,9 @@ def test_significance(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     output = sp.run(
         ["theta2_significance", "-d", "2020-01-17", "-s", "LST1"],

Diff for: src/osa/job.py

@@ -220,7 +220,10 @@ def historylevel(history_file: Path, data_type: str):
     exit_status = 0
 
     if history_file.exists():
-        match = re.search(r"sequence_LST1_(\d+)\.\d+", str(history_file))
+        if data_type == "DATA":
+            match = re.search(r"sequence_LST1_(\d+)\.\d+", str(history_file))
+        elif data_type == "PEDCALIB":
+            match = re.search(r"sequence_LST1_(\d+)\.history", str(history_file))
         run_id = int(match.group(1)) 
         for line in history_file.read_text().splitlines():
             words = line.split()

Diff for: src/osa/nightsummary/tests/test_extract.py

@@ -11,6 +11,9 @@ def test_source_list(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Test that the list of name of sources is correct."""
     from osa.nightsummary.extract import get_source_list

Diff for: src/osa/scripts/tests/test_osa_scripts.py

@@ -65,7 +65,10 @@ def test_simulate_processing(
     run_summary_file,
     r0_data,
     merged_run_summary,
-    drive_log
+    drive_log,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
 
     for file in drs4_time_calibration_files:
@@ -80,6 +83,9 @@ def test_simulate_processing(
     assert run_summary_file.exists()
     assert merged_run_summary.exists()
     assert drive_log.exists()
+    assert rf_models[1].exists()
+    assert dl1b_config_files[0].exists()
+    assert tailcuts_log_files[1].exists()
 
     remove_provlog()
     rc = run_program("simulate_processing", "-p", "--force")
@@ -129,6 +135,9 @@ def test_simulated_sequencer(
     r0_data,
     merged_run_summary,
     gain_selection_flag_file,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     assert run_summary_file.exists()
     assert run_catalog.exists()
@@ -186,6 +195,9 @@ def test_closer(
     longterm_dir,
     longterm_link_latest_dir,
     daily_datacheck_dl1_files,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     # First assure that the end of night flag is not set and remove it otherwise
     night_finished_flag = Path(
@@ -246,10 +258,11 @@ def test_datasequence(
     running_analysis_dir,
     run_catalog,
     run_catalog_dir,
-    rf_models_allsky_basedir,
-    rf_model_path,
+    rf_models_base_dir,
+    rf_models,
     catB_closed_file,
-    dl1b_config_file
+    dl1b_config_files,
+    tailcuts_log_files,
 ):
     drs4_file = "drs4_pedestal.Run00001.0000.fits"
     calib_file = "calibration.Run00002.0000.hdf5"
@@ -263,10 +276,10 @@ def test_datasequence(
 
     assert run_catalog_dir.exists()
     assert run_catalog.exists()
-    assert rf_models_allsky_basedir.exists()
-    assert rf_model_path.exists()
+    assert rf_models_base_dir.exists()
+    assert rf_models[1].exists()
     assert catB_closed_file.exists()
-    assert dl1b_config_file.exists()
+    assert dl1b_config_files[0].exists()
 
     output = run_program(
         "datasequence",
@@ -279,6 +292,10 @@ def test_datasequence(
         f"--systematic-correction-file={systematic_correction_file}",
         f"--drive-file={drive_file}",
         f"--run-summary={runsummary_file}",
+        f"--rf-model-path={rf_models[1]}",
+        f"--dl1b-config={dl1b_config_files[0]}",
+        "--dl1-prod-id=tailcut84",
+        "--dl2-prod-id=tailcut84/nsb_tuning_0.14",
         run_number,
         "LST1",
     )
@@ -303,7 +320,15 @@ def test_calibration_pipeline(running_analysis_dir):
     assert output.returncode == 0
 
 
-def test_is_sequencer_successful(run_summary, running_analysis_dir):
+def test_is_sequencer_successful(
+        run_summary,
+        running_analysis_dir,
+        dl1b_config_files,
+        tailcuts_log_files,
+        rf_models,
+        merged_run_summary,
+    ):
+    assert merged_run_summary.exists()
     options.directory = running_analysis_dir
     options.test = True
     seq_tuple = is_finished_check(run_summary)

Diff for: src/osa/tests/test_jobs.py

@@ -15,12 +15,14 @@
 options.prod_id = "v0.1.0"
 
 
-def test_historylevel():
+def test_historylevel(
+        run_catalog,
+        dl1b_config_files,
+        tailcuts_log_files,
+        rf_models,
+):
     from osa.job import historylevel
 
-    options.dl1_prod_id = "tailcut84"
-    options.dl2_prod_id = "model1"
-
     level, rc = historylevel(datasequence_history_file, "DATA")
     assert level == 0
     assert rc == 0
@@ -29,9 +31,6 @@ def test_historylevel():
     assert level == 0
     assert rc == 0
 
-    options.dl1_prod_id = "tailcut84"
-    options.dl2_prod_id = "model2"
-
     level, rc = historylevel(datasequence_history_file, "DATA")
     assert level == 1
     assert rc == 0
@@ -140,12 +139,13 @@ def test_create_job_template_scheduler(
     calibration_file,
     run_summary_file,
     pedestal_ids_file,
-    rf_model_path,
+    dl1b_config_files,
+    rf_models,
 ):
     from osa.job import data_sequence_job_template
 
     assert pedestal_ids_file.exists()
-    assert rf_model_path.exists()
+    assert rf_models[1].exists()
 
     options.test = False
     options.simulate = False
@@ -191,6 +191,8 @@ def test_create_job_template_scheduler(
             '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
             '--run-summary={run_summary_file}',
+            '--dl1b-config={dl1b_config_files[0]}',
+            '--dl1-prod-id=tailcut84',
             f'01807.{{subruns:04d}}',
             'LST1'
         ])
@@ -238,6 +240,8 @@ def test_create_job_template_scheduler(
                 '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
                 '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
                 '--run-summary={run_summary_file}',
+                '--dl1b-config={dl1b_config_files[1]}',
+                '--dl1-prod-id=tailcut84',
                 f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
                 f'01808.{{subruns:04d}}',
                 'LST1'
@@ -260,7 +264,8 @@ def test_create_job_template_local(
     run_summary_file,
     pedestal_ids_file,
     r0_data,
-    rf_model_path,
+    dl1b_config_files,
+    rf_models,
 ):
     """Check the job file in local mode (assuming no scheduler)."""
     from osa.job import data_sequence_job_template
@@ -275,7 +280,7 @@ def test_create_job_template_local(
         assert file.exists()
 
     assert pedestal_ids_file.exists()
-    assert rf_model_path.exists()
+    assert rf_models[0].exists()
 
     options.test = True
     options.simulate = False
@@ -307,7 +312,10 @@ def test_create_job_template_local(
             '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
             '--run-summary={run_summary_file}',
-            '--rf-model-path={rf_model_path}',
+            '--dl1b-config={dl1b_config_files[0]}',
+            '--dl1-prod-id=tailcut84',
+            '--rf-model-path={rf_models[1]}',
+            '--dl2-prod-id=tailcut84/nsb_tuning_0.14',
             f'01807.{{subruns:04d}}',
             'LST1'
         ])
@@ -341,7 +349,10 @@ def test_create_job_template_local(
                 '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
                 '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
                 '--run-summary={run_summary_file}',
-                '--rf-model-path={rf_model_path}',
+                '--dl1b-config={dl1b_config_files[1]}',
+                '--dl1-prod-id=tailcut84',
+                '--rf-model-path={rf_models[1]}',
+                '--dl2-prod-id=tailcut84/nsb_tuning_0.14',
                 f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
                 f'01808.{{subruns:04d}}',
                 'LST1'