Merge branch 'catB_option1' into dl2_production

Author: marialainez

Diff for: pyproject.toml

@@ -76,6 +76,7 @@ reprocess_longterm = "osa.scripts.reprocess_longterm:main"
 gain_selection = "osa.scripts.gain_selection:main"
 update_source_catalog = "osa.scripts.update_source_catalog:main"
 gainsel_webmaker = "osa.scripts.gainsel_webmaker:main"
+sequencer_catB_tailcuts = "osa.scripts.sequencer_catB_tailcuts:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

Diff for: src/osa/configs/options.py

@@ -15,6 +15,8 @@
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None

Diff for: src/osa/configs/sequencer.cfg

@@ -15,9 +15,11 @@ RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
@@ -46,18 +48,22 @@ DL2_PROD_ID: model2
 #charge_calibration: onsite_create_calibration_file
 drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file
 charge_calibration: lstcam_calib_onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
 r0_to_dl1: lstchain_data_r0_to_dl1
 dl1ab: lstchain_dl1ab
 check_dl1: lstchain_check_dl1
 dl1_to_dl2: lstchain_dl1_to_dl2
+tailcuts_finder: lstchain_find_tailcuts
 
 # To be set by the user
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: True
+apply_standard_dl1b_config: False
 use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json

Diff for: src/osa/conftest.py

@@ -596,3 +596,56 @@ def gain_selection_flag_file(osa_dir):
     file = GainSel_dir / "GainSelFinished.txt"
     file.touch()
     return file
+
+
+@pytest.fixture(scope="session")
+def catB_closed_file(running_analysis_dir):
+
+    catB_closed_file = running_analysis_dir / "catB_00003.closed"
+    catB_closed_file.touch()
+    return catB_closed_file
+
+
+@pytest.fixture(scope="session")
+def catB_calib_base_dir(monitoring_dir):
+
+    catB_calib_base_dir = monitoring_dir / "PixelCalibration" / "Cat-B"
+    catB_calib_base_dir.mkdir(parents=True, exist_ok=True)
+    return catB_calib_base_dir
+
+
+@pytest.fixture(scope="session")
+def catB_calibration_file(catB_calib_dir):
+
+    catB_calib_dir = catB_calib_base_dir / "calibration" / nightdir / prod_id
+    catB_calib_file = catB_calib_dir / "cat_B_calibration_filters_52.Run00003.h5"
+    catB_calib_file.touch()
+    return catB_calib_file
+
+
+@pytest.fixture(scope="session")
+def dl1b_config_file(running_analysis_dir):
+    config_information = dedent(
+        """\
+            {
+            "tailcuts_clean_with_pedestal_threshold": {
+                "picture_thresh": 8,
+                "boundary_thresh": 4,
+                "sigma": 2.5,
+                "keep_isolated_pixels": false,
+                "min_number_picture_neighbors": 2,
+                "use_only_main_island": false,
+                "delta_time": 2
+            },
+            "dynamic_cleaning": {
+                "apply": true,
+                "threshold": 267,
+                "fraction_cleaning_intensity": 0.03
+            }
+        }"""
+    )
+    config_file = running_analysis_dir / "dl1ab_Run00003.json"
+    config_file.touch()
+    config_file.write_text(config_information)
+    return config_file
+

Diff for: src/osa/job.py

@@ -99,6 +99,14 @@ def are_all_jobs_correctly_finished(sequence_list):
     analysis_directory = Path(options.directory)
     for sequence in sequence_list:
         history_files_list = analysis_directory.rglob(f"*{sequence.seq}*.history")
+        
+        if not options.test:
+            try:
+                next(history_files_list)
+            except StopIteration:
+                log.info("no history files found")
+                flag = False
+
         for history_file in history_files_list:
             # TODO: s.history should be SubRunObj attribute not RunObj
             # s.history only working for CALIBRATION sequence (run-wise), since it is
@@ -115,6 +123,12 @@ def are_all_jobs_correctly_finished(sequence_list):
                     f"finished up to DL1ab, but --no-dl2 option selected"
                 )
                 continue
+            if out == 3 and options.no_dl1ab:
+                log.debug(
+                    f"Job {sequence.seq} ({sequence.type}) correctly "
+                    f"finished up to DL1A, but --no-dl1ab option selected"
+                )
+                continue
 
             log.warning(
                 f"Job {sequence.seq} (run {sequence.run}) not correctly finished [level {out}]"
@@ -221,6 +235,8 @@ def historylevel(history_file: Path, data_type: str):
                 # Data sequence
                 elif program == cfg.get("lstchain", "r0_to_dl1"):
                     level = 3 if exit_status == 0 else 4
+                #elif program == cfg.get("lstchain", "catB_calibration"):
+                #    level = 3 if exit_status == 0 else 4
                 elif program == cfg.get("lstchain", "dl1ab"):
                     if (exit_status == 0) and (prod_id == options.dl1_prod_id):
                         log.debug(f"DL1ab prod ID: {options.dl1_prod_id} already produced")
@@ -431,6 +447,8 @@ def data_sequence_job_template(sequence):
         commandargs.extend(("--config", f"{Path(options.configfile).resolve()}"))
     if sequence.type == "DATA" and options.no_dl2:
         commandargs.append("--no-dl2")
+    if sequence.type == "DATA" and options.no_dl1ab:
+        commandargs.append("--no-dl1ab")
 
     commandargs.extend(
         (

Diff for: src/osa/paths.py

@@ -7,12 +7,11 @@
 from typing import List
 import subprocess
 import time
-
+import json
 import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file,
-                             find_filter_wheels)
+                             find_time_calibration_file)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -45,8 +44,8 @@
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
@@ -136,18 +135,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
-
-    if options.test:  # Run tests avoiding the access to the database
-        options.filters = 52
-
-    else:
-        mongodb = cfg.get("database", "caco_db")
-        try:
-            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = find_filter_wheels(int(run_id), mongodb)
-        except IOError:
-            log.warning("No filter information found in database. Assuming positions 52.")
-            options.filters = 52
+    options.filters = utils.get_calib_filters(run_id)
 
     return (
         CALIB_BASEDIR
@@ -156,6 +144,15 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
     ).resolve()
 
 
+def get_catB_calibration_filename(run_id: int) -> Path:
+    """Return the Category-B calibration filename of a given run."""
+    date = utils.date_to_dir(options.date)
+    calib_prod_id = utils.get_lstchain_version()
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id
+    filters = utils.get_calib_filters(run_id)
+    return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+
+
 def pedestal_ids_file_exists(run_id: int) -> bool:
     """Look for the files with pedestal interleaved event identification."""
     pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR"))
@@ -405,3 +402,27 @@ def dl1_datacheck_longterm_file_exits() -> bool:
     longterm_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5"
     return longterm_file.exists()
 
+
+def catB_closed_file_exists(run_id: int) -> bool:
+    catB_closed_file = Path(options.directory) / f"catB_{run_id:05d}.closed"
+    return catB_closed_file.exists()
+
+
+def catB_calibration_file_exists(run_id: int) -> bool:
+    catB_calib_base_dir = Path(cfg.get("LST1","CAT_B_CALIB_BASE"))
+    prod_id = utils.get_lstchain_version()
+    night_dir = utils.date_to_dir(options.date)
+    filters = utils.get_calib_filters(run_id)
+    catB_calib_dir = catB_calib_base_dir / "calibration" / night_dir / prod_id 
+    catB_calib_file = catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+    return catB_calib_file.exists()
+
+
+def get_dl1_prod_id(config_filename):
+    with open(config_filename) as json_file:
+        data = json.load(json_file)
+        
+    picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
+    boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+    
+    return f"tailcut{picture_thresh}{boundary_thresh}"

Diff for: src/osa/provenance/capture.py

@@ -53,7 +53,7 @@
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}

Diff for: src/osa/provenance/config/definition.yaml

@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"

Diff for: src/osa/provenance/utils.py

@@ -10,7 +10,7 @@
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -43,8 +43,8 @@ def parse_variables(class_instance):
     rf_models_directory = Path(cfg.get("LST1", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
@@ -133,6 +133,9 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+  
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000

Diff for: src/osa/scripts/calibration_pipeline.py

@@ -44,23 +44,27 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "drs4_baseline")
     return [
         command,
         "-r", str(drs4_pedestal_run_id),
         "-b", base_dir,
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "charge_calibration")
     cmd = [
         command,
         "-p", str(drs4_pedestal_run_id),
         "-r", str(pedcal_run_id),
         "-b", base_dir,
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):