Skip to content

Commit 3e64d93

Browse files
jtmccannjtmccann
committed
Add amplifier bias, dark, and flat percentile plots.
1 parent 6b09da1 commit 3e64d93

7 files changed

Lines changed: 405 additions & 0 deletions

File tree

‎pipelines/amplifierQualityCore.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
description: |
2+
Percentile plots for each amplifier on a detector for a bias, dark, and flat.
3+
tasks:
4+
amplifierAnalysis:
5+
class: lsst.analysis.tools.tasks.amplifierAnalysis.AmplifierAnalysisTask
6+
config:
7+
atools.biasPercentilePlot: BiasPercentilePlot
8+
atools.darkPercentilePlot: DarkPercentilePlot
9+
atools.flatPercentilePlot: FlatPercentilePlot
10+
python: |
11+
from lsst.analysis.tools.atools import *

‎python/lsst/analysis/tools/actions/plot/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from .focalPlanePlot import *
66
from .histPlot import *
77
from .multiVisitCoveragePlot import *
8+
from .percentilePlot import *
89
from .propertyMapPlot import *
910
from .rhoStatisticsPlot import *
1011
from .scatterplotWithTwoHists import *

‎python/lsst/analysis/tools/actions/plot/percentilePlot.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# This file is part of analysis_tools.
2+
#
3+
# Developed for the LSST Data Management System.
4+
# This product includes software developed by the LSST Project
5+
# (https://www.lsst.org).
6+
# See the COPYRIGHT file at the top-level directory of this distribution
7+
# for details of code ownership.
8+
#
9+
# This program is free software: you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation, either version 3 of the License, or
12+
# (at your option) any later version.
13+
#
14+
# This program is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License
20+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
21+
22+
23+
from ...interfaces import KeyedData, KeyedDataSchema, PlotAction, Scalar, ScalarType, Vector
24+
from astropy.table import vstack
25+
from matplotlib.figure import Figure
26+
import matplotlib.pyplot as plt
27+
import numpy as np
28+
from .plotUtils import addPlotInfo
29+
from typing import Mapping
30+
31+
__all__ = ("PercentilePlot",)
32+
33+
34+
class PercentilePlot(PlotAction):
35+
"""Makes a scatter plot of the data with a marginal
36+
histogram for each axis.
37+
"""
38+
39+
def getInputSchema(self) -> KeyedDataSchema:
40+
base: list[tuple[str, type[Vector] | ScalarType]] = []
41+
base.append(("amplifier", Vector))
42+
base.append(("detector", Vector))
43+
base.append(("percentile_0", Vector))
44+
base.append(("percentile_5", Vector))
45+
base.append(("percentile_16", Vector))
46+
base.append(("percentile_50", Vector))
47+
base.append(("percentile_84", Vector))
48+
base.append(("percentile_95", Vector))
49+
base.append(("percentile_100", Vector))
50+
return base
51+
52+
def __call__(self, data: KeyedData, **kwargs) -> Mapping[str, Figure] | Figure:
53+
self._validateInput(data, **kwargs)
54+
return self.makePlot(data, **kwargs)
55+
56+
def _validateInput(self, data: KeyedData, **kwargs) -> None:
57+
"""NOTE currently can only check that something is not a Scalar, not
58+
check that the data is consistent with Vector
59+
"""
60+
needed = self.getFormattedInputSchema(**kwargs)
61+
if remainder := {key.format(**kwargs) for key, _ in needed} - {
62+
key.format(**kwargs) for key in data.keys()
63+
}:
64+
raise ValueError(f"Task needs keys {remainder} but they were not found in input")
65+
for name, typ in needed:
66+
isScalar = issubclass((colType := type(data[name.format(**kwargs)])), Scalar)
67+
if isScalar and typ != Scalar:
68+
raise ValueError(f"Data keyed by {name} has type {colType} but action requires type {typ}")
69+
70+
def makePlot(self, data, plotInfo, **kwargs):
71+
"""Makes a plot showing the percentiles of the normalized distribution
72+
of the data.
73+
74+
Parameters
75+
----------
76+
data : `KeyedData`
77+
All the data
78+
plotInfo : `dict`
79+
A dictionary of information about the data being plotted with keys:
80+
``camera``
81+
The camera used to take the data (`lsst.afw.cameraGeom.Camera`)
82+
``"cameraName"``
83+
The name of camera used to take the data (`str`).
84+
``"filter"``
85+
The filter used for this data (`str`).
86+
``"ccdKey"``
87+
The ccd/dectector key associated with this camera (`str`).
88+
``"visit"``
89+
The visit of the data; only included if the data is from a
90+
single epoch dataset (`str`).
91+
``"patch"``
92+
The patch that the data is from; only included if the data is
93+
from a coadd dataset (`str`).
94+
``"tract"``
95+
The tract that the data comes from (`str`).
96+
``"photoCalibDataset"``
97+
The dataset used for the calibration, e.g. "jointcal" or "fgcm"
98+
(`str`).
99+
``"skyWcsDataset"``
100+
The sky Wcs dataset used (`str`).
101+
``"rerun"``
102+
The rerun the data is stored in (`str`).
103+
104+
Returns
105+
------
106+
``fig``
107+
The figure to be saved (`matplotlib.figure.Figure`).
108+
109+
Notes
110+
-----
111+
Makes a plot showing the normalized percentile distribution of data.
112+
"""
113+
amplifiers = [
114+
"C17",
115+
"C07",
116+
"C16",
117+
"C06",
118+
"C15",
119+
"C05",
120+
"C14",
121+
"C04",
122+
"C13",
123+
"C03",
124+
"C12",
125+
"C02",
126+
"C11",
127+
"C01",
128+
"C10",
129+
"C00",
130+
]
131+
# TODO: generalize to make N per-detector plots
132+
detector = data["detector"] == 0
133+
data = vstack([data[detector & (data["amplifier"] == amp)][0] for amp in amplifiers])
134+
percentiles = ["0", "5", "16", "50", "84", "95", "100"]
135+
distributions = [data[f"percentile_{pct}"] for pct in percentiles]
136+
medians = [np.nanmedian(dist) for dist in distributions]
137+
normalizedDistributions = [np.abs(dist / med) for (med, dist) in list(zip(medians, distributions))]
138+
139+
fig, axs = plt.subplots(nrows=8, ncols=2, sharex=True, sharey=True)
140+
# Set threshold for a bad normalized bias.
141+
threshold = [0.1, 10]
142+
pcts = [int(pct) for pct in percentiles]
143+
for i, ax in enumerate(axs.reshape(16)):
144+
distribution = np.array([dist[i] for dist in normalizedDistributions])
145+
colors = np.where((distribution < threshold[0]) | (distribution > threshold[1]), "r", "C0")
146+
ax.hlines(1.0, xmin=pcts[0], xmax=pcts[-1], colors="k", linestyle="--")
147+
ax.scatter(pcts, distribution, c=colors)
148+
ax.plot(pcts, distribution)
149+
ax.set_ylabel(data["amplifier"][i])
150+
ax.set_yscale("log")
151+
152+
plt.xticks(ticks=pcts, labels=percentiles)
153+
fig.supxlabel("Percentile")
154+
fig.supylabel("Normalized distribution")
155+
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0)
156+
157+
# Add useful information to the plot
158+
fig = plt.gcf()
159+
addPlotInfo(fig, plotInfo)
160+
return fig

‎python/lsst/analysis/tools/atools/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from .amplifierPercentilePlots import *
12
from .astrometricRepeatability import *
23
from .coveragePlots import *
34
from .deblenderMetric import *

‎python/lsst/analysis/tools/atools/amplifierPercentilePlots.py‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
# This file is part of analysis_tools.
2+
#
3+
# Developed for the LSST Data Management System.
4+
# This product includes software developed by the LSST Project
5+
# (https://www.lsst.org).
6+
# See the COPYRIGHT file at the top-level directory of this distribution
7+
# for details of code ownership.
8+
#
9+
# This program is free software: you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation, either version 3 of the License, or
12+
# (at your option) any later version.
13+
#
14+
# This program is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License
20+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
21+
from __future__ import annotations
22+
23+
__all__ = (
24+
"BiasPercentilePlot",
25+
"DarkPercentilePlot",
26+
"FlatPercentilePlot",
27+
)
28+
29+
from ..actions.plot.percentilePlot import PercentilePlot
30+
31+
# from ..actions.scalar.scalarActions import MedianAction, SigmaMadAction
32+
from ..actions.vector import LoadVector
33+
from ..interfaces import AnalysisTool
34+
35+
36+
class BiasPercentilePlot(AnalysisTool):
37+
"""Plot the percentiles of the normalized amplifier bias distributions."""
38+
39+
def setDefaults(self):
40+
super().setDefaults()
41+
self.process.buildActions.amplifier = LoadVector()
42+
self.process.buildActions.amplifier.vectorKey = "amplifier"
43+
44+
self.process.buildActions.detector = LoadVector()
45+
self.process.buildActions.detector.vectorKey = "detector"
46+
47+
self.process.buildActions.percentile_0 = LoadVector()
48+
self.process.buildActions.percentile_0.vectorKey = "biasDistribution_0.0"
49+
50+
self.process.buildActions.percentile_5 = LoadVector()
51+
self.process.buildActions.percentile_5.vectorKey = "biasDistribution_5.0"
52+
53+
self.process.buildActions.percentile_16 = LoadVector()
54+
self.process.buildActions.percentile_16.vectorKey = "biasDistribution_16.0"
55+
56+
self.process.buildActions.percentile_50 = LoadVector()
57+
self.process.buildActions.percentile_50.vectorKey = "biasDistribution_50.0"
58+
59+
self.process.buildActions.percentile_84 = LoadVector()
60+
self.process.buildActions.percentile_84.vectorKey = "biasDistribution_84.0"
61+
62+
self.process.buildActions.percentile_95 = LoadVector()
63+
self.process.buildActions.percentile_95.vectorKey = "biasDistribution_95.0"
64+
65+
self.process.buildActions.percentile_100 = LoadVector()
66+
self.process.buildActions.percentile_100.vectorKey = "biasDistribution_100.0"
67+
68+
# self.process.calculateActions.mag50 = Mag50Action()
69+
# self.process.calculateActions.mag50.vectorKey = "{band}_mag_ref"
70+
# self.process.calculateActions.mag50.matchDistanceKey = "matchDistance"
71+
72+
self.produce.plot = PercentilePlot()
73+
# self.produce.metric.units = {"mag50": "mag"}
74+
# self.produce.metric.newNames = {"mag50": "{band}_mag50"}
75+
76+
77+
class DarkPercentilePlot(AnalysisTool):
78+
"""Plot the percentiles of the normalized amplifier dark distributions."""
79+
80+
def setDefaults(self):
81+
super().setDefaults()
82+
self.process.buildActions.amplifier = LoadVector()
83+
self.process.buildActions.amplifier.vectorKey = "amplifier"
84+
85+
self.process.buildActions.detector = LoadVector()
86+
self.process.buildActions.detector.vectorKey = "detector"
87+
88+
self.process.buildActions.percentile_0 = LoadVector()
89+
self.process.buildActions.percentile_0.vectorKey = "darkDistribution_0.0"
90+
91+
self.process.buildActions.percentile_5 = LoadVector()
92+
self.process.buildActions.percentile_5.vectorKey = "darkDistribution_5.0"
93+
94+
self.process.buildActions.percentile_16 = LoadVector()
95+
self.process.buildActions.percentile_16.vectorKey = "darkDistribution_16.0"
96+
97+
self.process.buildActions.percentile_50 = LoadVector()
98+
self.process.buildActions.percentile_50.vectorKey = "darkDistribution_50.0"
99+
100+
self.process.buildActions.percentile_84 = LoadVector()
101+
self.process.buildActions.percentile_84.vectorKey = "darkDistribution_84.0"
102+
103+
self.process.buildActions.percentile_95 = LoadVector()
104+
self.process.buildActions.percentile_95.vectorKey = "darkDistribution_95.0"
105+
106+
self.process.buildActions.percentile_100 = LoadVector()
107+
self.process.buildActions.percentile_100.vectorKey = "darkDistribution_100.0"
108+
109+
# self.process.calculateActions.mag50 = Mag50Action()
110+
# self.process.calculateActions.mag50.vectorKey = "{band}_mag_ref"
111+
# self.process.calculateActions.mag50.matchDistanceKey = "matchDistance"
112+
113+
self.produce.plot = PercentilePlot()
114+
# self.produce.metric.units = {"mag50": "mag"}
115+
# self.produce.metric.newNames = {"mag50": "{band}_mag50"}
116+
117+
118+
class FlatPercentilePlot(AnalysisTool):
119+
"""Plot the percentiles of the normalized amplifier flat distributions."""
120+
121+
def setDefaults(self):
122+
super().setDefaults()
123+
self.process.buildActions.amplifier = LoadVector()
124+
self.process.buildActions.amplifier.vectorKey = "amplifier"
125+
126+
self.process.buildActions.detector = LoadVector()
127+
self.process.buildActions.detector.vectorKey = "detector"
128+
129+
self.process.buildActions.percentile_0 = LoadVector()
130+
self.process.buildActions.percentile_0.vectorKey = "flatDistribution_0.0"
131+
132+
self.process.buildActions.percentile_5 = LoadVector()
133+
self.process.buildActions.percentile_5.vectorKey = "flatDistribution_5.0"
134+
135+
self.process.buildActions.percentile_16 = LoadVector()
136+
self.process.buildActions.percentile_16.vectorKey = "flatDistribution_16.0"
137+
138+
self.process.buildActions.percentile_50 = LoadVector()
139+
self.process.buildActions.percentile_50.vectorKey = "flatDistribution_50.0"
140+
141+
self.process.buildActions.percentile_84 = LoadVector()
142+
self.process.buildActions.percentile_84.vectorKey = "flatDistribution_84.0"
143+
144+
self.process.buildActions.percentile_95 = LoadVector()
145+
self.process.buildActions.percentile_95.vectorKey = "flatDistribution_95.0"
146+
147+
self.process.buildActions.percentile_100 = LoadVector()
148+
self.process.buildActions.percentile_100.vectorKey = "flatDistribution_100.0"
149+
150+
# self.process.calculateActions.mag50 = Mag50Action()
151+
# self.process.calculateActions.mag50.vectorKey = "{band}_mag_ref"
152+
# self.process.calculateActions.mag50.matchDistanceKey = "matchDistance"
153+
154+
self.produce.plot = PercentilePlot()
155+
# self.produce.metric.units = {"mag50": "mag"}
156+
# self.produce.metric.newNames = {"mag50": "{band}_mag50"}

‎python/lsst/analysis/tools/tasks/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from .amplifierAnalysis import *
12
from .assocDiaSrcDetectorVisitAnalysis import *
23
from .associatedSourcesTractAnalysis import *
34
from .astrometricCatalogMatch import *

0 commit comments

Comments
 (0)