Initial commit for NDLAr flow workflow.

scripts/ndlar_scripts/get_ndlar_input.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+#DATA_DIR=$1
+DATA_DIR="../../data/ndlar_flow/"
+mkdir -p $DATA_DIR
+
+HERE=`pwd`
+
+cd ${DATA_DIR}
+
+# ndlar layout describing a *single* module (fix me)
+curl -O https://raw.githubusercontent.com/DUNE/larnd-sim/develop/larndsim/pixel_layouts/multi_tile_layout-3.0.40.yaml
+
+# ndlar detector description
+curl -O https://raw.githubusercontent.com/DUNE/larnd-sim/develop/larndsim/detector_properties/ndlar-module.yaml
+
+# Placeholder
+curl -O https://portal.nersc.gov/project/dune/data/2x2/simulation/kwood_dev/proto_nd_flow_inputs/runlist-2x2-mcexample.txt
+
+cd ${HERE}

scripts/ndlar_scripts/run_ndlar_flow_example.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Runs ndlar_flow on an example file.
+# Before using this script, use
+# >> source get_ndlar_input.sh
+# to download all the necessary inputs into the correct directories
+#
+
+INPUT_FILE=$1
+
+OUTPUT_DIR=`pwd` #!!! change me
+OUTPUT_NAME=(${INPUT_FILE//"/"/ })
+OUTPUT_NAME=${OUTPUT_NAME[-1]}
+OUTPUT_FILE="${OUTPUT_DIR}/${OUTPUT_NAME}"
+OUTPUT_FILE=${OUTPUT_FILE//.hdf5/.ndlar_flow.hdf5}
+echo ${OUTPUT_FILE}
+
+# for running on a login node
+H5FLOW_CMD='h5flow'
+# for running on a single compute node with 32 cores
+#H5FLOW_CMD='srun -n32 h5flow'
+
+# run all stages
+WORKFLOW1='yamls/ndlar_flow/workflows/charge/charge_event_building.yaml'
+WORKFLOW2='yamls/ndlar_flow/workflows/charge/charge_event_reconstruction.yaml'
+WORKFLOW3='yamls/ndlar_flow/workflows/combined/combined_reconstruction.yaml'
+WORKFLOW4='yamls/ndlar_flow/workflows/charge/prompt_calibration.yaml'
+WORKFLOW5='yamls/ndlar_flow/workflows/charge/final_calibration.yaml'
+
+HERE=`pwd`
+#cd ndlar_flow
+# assumes this is being run from ndlar_flow/scripts/ndlar_flow:
+cd ../../
+
+# avoid being asked if we want to overwrite the file if it exists.
+# this is us answering "yes".
+if [ -e $OUTPUT_FILE ]; then
+    rm $OUTPUT_FILE
+fi
+
+$H5FLOW_CMD -c $WORKFLOW1 $WORKFLOW2 $WORKFLOW3 $WORKFLOW4 $WORKFLOW5 -i $INPUT_FILE -o $OUTPUT_FILE
+
+echo "Done!"
+echo "Output can be found at $OUTPUT_FILE"
+
+cd ${HERE}
+

yamls/ndlar_flow/resources/Geometry.yaml

@@ -0,0 +1,11 @@
+classname: Geometry # resources/geometry.py
+path: resources.geometry
+params:
+  path: 'geometry_info'
+  det_geometry_file: 'data/ndlar_flow/ndlar-module.yaml'
+  crs_geometry_files: ['data/ndlar_flow/multi_tile_layout-3.0.40.yaml'] 
+  # TO DO: what is the line below.
+  crs_geometry_to_module: [0,0,1,0]
+  #lrs_geometry_file: 'data/proto_nd_flow/light_module_desc-2.0.0.yaml'
+  beam_direction: 'z'
+  drift_direction: 'x'

yamls/ndlar_flow/resources/RunData.yaml

@@ -0,0 +1,14 @@
+classname: RunData
+path: proto_nd_flow.resources.run_data
+params:
+  path: 'run_info'
+  runlist_file: 'data/proto_nd_flow/runlist-2x2-mcexample.txt'
+  defaults:
+    charge_filename: '-'
+    light_filename: '-'
+    e_field: 0.05 # kV/mm
+    light_samples: 256
+    charge_thresholds: 'medm'
+    is_mc: True
+    crs_ticks: 0.1 # us
+    lrs_ticks: 0.016 # us

yamls/ndlar_flow/workflows/charge/charge_event_building.yaml

@@ -0,0 +1,16 @@
+# Generates the low-level event built data for charge data (i.e. grouped
+# packets)
+
+flow:
+  source: raw_event_generator
+  stages: [] # just generate events, no analysis stages
+  drop: []
+
+
+resources:
+  - !include yamls/ndlar_flow/resources/RunData.yaml
+
+
+raw_event_generator: # groups time-sorted data packets from larpix datalog files
+  !include yamls/proto_nd_flow/reco/charge/RawEventGenerator.yaml
+

yamls/ndlar_flow/workflows/charge/charge_event_reconstruction.yaml

@@ -0,0 +1,37 @@
+# Generates the mid-level event built data for charge data (i.e. hits and
+# external triggers)
+
+flow:
+  source: raw_events
+  stages: [timestamp_corrector, ext_trig_finder, raw_hit_builder, event_builder]
+  drop: []
+
+
+resources:
+  - !include yamls/ndlar_flow/resources/RunData.yaml
+  - !include yamls/ndlar_flow/resources/Geometry.yaml
+
+
+raw_events:
+  classname: H5FlowDatasetLoopGenerator
+  path: h5flow.modules
+  dset_name: 'charge/raw_events'
+  params:
+    chunk_size: 128
+
+
+timestamp_corrector:
+  !include yamls/proto_nd_flow/reco/charge/TimestampCorrector.yaml
+
+
+ext_trig_finder:
+  !include yamls/proto_nd_flow/reco/charge/ExternalTriggerFinder.yaml
+
+
+raw_hit_builder:
+  !include yamls/proto_nd_flow/reco/charge/RawHitBuilder.yaml
+
+
+event_builder:
+  !include yamls/proto_nd_flow/reco/charge/EventBuilder.yaml
+

yamls/ndlar_flow/workflows/charge/final_calibration.yaml

@@ -0,0 +1,26 @@
+# Generates the mid-level event built data for charge data (i.e. hits and
+# external triggers)
+
+flow:
+  source: raw_events
+  #source: calib_prompt_hits
+  #stages: [temp_hit_builder, calib_hit_merger]
+  stages: [calib_hit_merger]
+  drop: []
+
+
+resources:
+  - !include yamls/ndlar_flow/resources/RunData.yaml
+  - !include yamls/proto_nd_flow/resources/LArData.yaml
+  - !include yamls/ndlar_nd_flow/resources/Geometry.yaml
+
+raw_events:
+  classname: H5FlowDatasetLoopGenerator
+  path: h5flow.modules
+  dset_name: 'charge/raw_events'
+  params:
+    chunk_size: 32
+
+calib_hit_merger:
+  !include yamls/proto_nd_flow/reco/charge/CalibHitMerger.yaml
+

yamls/ndlar_flow/workflows/charge/prompt_calibration.yaml

@@ -0,0 +1,25 @@
+# Generates the mid-level event built data for charge data (i.e. hits and
+# external triggers)
+
+flow:
+  source: raw_events
+  stages: [calib_hit_builder]
+  drop: []
+
+
+resources:
+  - !include yamls/ndlar_flow/resources/RunData.yaml
+  - !include yamls/proto_nd_flow/resources/LArData.yaml
+  - !include yamls/ndlar_flow/resources/Geometry.yaml
+
+
+raw_events:
+  classname: H5FlowDatasetLoopGenerator
+  path: h5flow.modules
+  dset_name: 'charge/raw_events'
+  params:
+    chunk_size: 128
+
+calib_hit_builder:
+  !include yamls/proto_nd_flow/reco/charge/CalibHitBuilder.yaml
+

yamls/ndlar_flow/workflows/combined/combined_reconstruction.yaml

@@ -0,0 +1,24 @@
+# Performs combined reconstruction, generating t0s, tracklets, ...
+
+flow:
+  source: events
+  #stages: [t0_reco, drift_reco, electron_lifetime_corr, tracklet_reco, tracklet_merge]
+  stages: [t0_reco]
+  drop: []
+
+events:
+  classname: H5FlowDatasetLoopGenerator
+  path: h5flow.modules
+  dset_name: 'charge/events'
+  params:
+    chunk_size: 32
+
+resources:
+  - !include yamls/ndlar_flow/resources/RunData.yaml
+  - !include yamls/ndlar_flow/resources/Geometry.yaml
+  - !include yamls/proto_nd_flow/resources/LArData.yaml
+#  - !include yamls/proto_nd_flow/resources/DisabledChannels.yaml
+
+t0_reco:
+  !include yamls/proto_nd_flow/reco/combined/T0Reconstruction.yaml
+