diff --git a/CHANGES.rst b/CHANGES.rst index eb25ae7a58..101a221acb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -496,6 +496,8 @@ outlier_detection - Fix errors in documentation describing arguments. [#8603] +- Re-enabled saving of blot models when `save_intermediate_results` is True. [#8758] + pathloss -------- diff --git a/docs/jwst/outlier_detection/outlier_detection_imaging.rst b/docs/jwst/outlier_detection/outlier_detection_imaging.rst index 15df877939..c2cd6318b9 100644 --- a/docs/jwst/outlier_detection/outlier_detection_imaging.rst +++ b/docs/jwst/outlier_detection/outlier_detection_imaging.rst @@ -75,7 +75,8 @@ Specifically, this routine performs the following operations: * The median image is written out to disk as `__median.fits` by default. #. By default, the median image is blotted back (inverse of resampling) to - match each original input image. + match each original input image. Resampled/blotted images are written out to disk if + the ``save_intermediate_results`` parameter is set to `True`. * **If resampling is turned off**, the median image is compared directly to each input image. diff --git a/docs/jwst/outlier_detection/outlier_detection_spec.rst b/docs/jwst/outlier_detection/outlier_detection_spec.rst index f66df77854..6d5b432b2f 100644 --- a/docs/jwst/outlier_detection/outlier_detection_spec.rst +++ b/docs/jwst/outlier_detection/outlier_detection_spec.rst @@ -39,6 +39,8 @@ Specifically, this routine performs the following operations (modified from the if the ``save_intermediate_results`` parameter is set to `True` #. Blot median image to match each original input image + - Resampled/blotted images are written out to disk if the ``save_intermediate_results`` + parameter is set to `True` - **If resampling is turned off**, the median image is used for comparison with the original input models for detecting outliers #. Perform statistical comparison between blotted image and original image to identify outliers diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py index 559db8cec0..9f1aabf597 100644 --- a/jwst/outlier_detection/imaging.py +++ b/jwst/outlier_detection/imaging.py @@ -127,7 +127,16 @@ def detect_outliers( with input_models: for image in input_models: if resample_data: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg) + flag_resampled_model_crs(image, + median_data, + median_wcs, + snr1, + snr2, + scale1, + scale2, + backg, + save_blot=save_intermediate_results, + make_output_path=make_output_path) else: flag_model_crs(image, median_data, snr1) input_models.shelve(image, modify=True) diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py index 74a21965a4..c482aad8b9 100644 --- a/jwst/outlier_detection/spec.py +++ b/jwst/outlier_detection/spec.py @@ -130,7 +130,11 @@ def detect_outliers( scale1, scale2, backg, + save_blot=save_intermediate_results, + make_output_path=make_output_path ) else: - flag_crs_in_models(input_models, median_data, snr1) + flag_crs_in_models(input_models, + median_data, + snr1) return input_models diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index b3b8c9b803..ebd732437b 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -288,11 +288,13 @@ def test_outlier_step_spec(tmp_cwd, tmp_path): i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) # intermediate files are removed assert len(i2d_files) == 0 assert len(s2d_files) == 0 assert len(median_files) == 0 + assert len(blot_files) == 0 # result files are written to the output directory if dirname == output_dir: @@ -320,25 +322,31 @@ def test_outlier_step_spec(tmp_cwd, tmp_path): i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) if dirname == output_dir: # result files are written to the output directory assert len(result_files) == len(container) - # s2d and median files are written to the output directory + # s2d, median, and blot files are written to the output directory assert len(s2d_files) == len(container) + assert len(blot_files) == len(container) assert len(median_files) == 1 # i2d files not written assert len(i2d_files) == 0 # nothing else was written - assert len(all_files) == len(s2d_files) + len(median_files) + len(result_files) + assert len(all_files) == len(s2d_files) + \ + len(median_files) + \ + len(result_files) + \ + len(blot_files) else: # nothing should be written to the current directory assert len(result_files) == 0 assert len(s2d_files) == 0 assert len(median_files) == 0 assert len(i2d_files) == 0 + assert len(blot_files) == 0 assert len(all_files) == 0 miri_rate.close() @@ -405,6 +413,30 @@ def test_outlier_step_on_disk(three_sci_as_asn, tmp_cwd): assert zeroth.dq[12, 12] == OUTLIER_DO_NOT_USE result.shelve(zeroth, modify=False) + # Verify intermediate results were written to disk + dirname = tmp_cwd + all_files = glob(os.path.join(dirname, '*.fits')) + input_files = glob(os.path.join(dirname, '*_cal.fits')) + result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) + s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) + median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) + + assert len(result_files) == len(container) + + # i2d, median, blot files are written to the output directory + assert len(i2d_files) == len(container) + assert len(blot_files) == len(container) + assert len(median_files) == 1 + + # s2d files not written + assert len(s2d_files) == 0 + + # nothing else was written + assert len(all_files) == len(input_files) + len(i2d_files) + len(median_files) + len(result_files) + len(blot_files) + + def test_outlier_step_square_source_no_outliers(we_three_sci, tmp_cwd): """Test whole step with square source with sharp edges, no outliers""" diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index 6491415715..4abdfbad1b 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -206,6 +206,8 @@ def flag_resampled_model_crs( scale1, scale2, backg, + save_blot=False, + make_output_path=None, ): if 'SPECTRAL' not in input_model.meta.wcs.output_frame.axes_type: input_pixflux_area = input_model.meta.photometry.pixelarea_steradians @@ -217,6 +219,15 @@ def flag_resampled_model_crs( pix_ratio = 1.0 blot = gwcs_blot(median_data, median_wcs, input_model.data.shape, input_model.meta.wcs, pix_ratio) + if save_blot: + if make_output_path is None: + raise ValueError("make_output_path must be provided if save_blot is True") + model_path = make_output_path(input_model.meta.filename, suffix='blot') + blot_model = _make_blot_model(input_model, blot) + blot_model.meta.filename = model_path + blot_model.save(model_path) + log.info(f"Saved model in {model_path}") + del blot_model # dq flags will be updated in-place _flag_resampled_model_crs(input_model, blot, snr1, snr2, scale1, scale2, backg) @@ -260,9 +271,20 @@ def flag_crs_in_models_with_resampling( scale1, scale2, backg, + save_blot=False, + make_output_path=None, ): for image in input_models: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg) + flag_resampled_model_crs(image, + median_data, + median_wcs, + snr1, + snr2, + scale1, + scale2, + backg, + save_blot=save_blot, + make_output_path=make_output_path) def flag_model_crs(image, blot, snr): @@ -276,3 +298,10 @@ def flag_model_crs(image, blot, snr): match_nans_and_flags(image) log.info(f"{np.count_nonzero(cr_mask)} pixels marked as outliers") + + +def _make_blot_model(input_model, blot): + blot_model = type(input_model)() + blot_model.data = blot + blot_model.update(input_model) + return blot_model \ No newline at end of file