Skip to content

Commit 9078f61

Browse files
weatherhead99weatherhead99
committed
Support multiple quantities plotting in CalibAmpScatterTool
This commit changes the `quantityKey` field in the `CalibAmpScatterTool` class to be of type `ListField`. This enables multiple quantities to be plotted on a single scatter plot. In addition, pipeline YAML files for calibration verification are re-worded to use the new `ListField` type
1 parent cdf17bb commit 9078f61

4 files changed

Lines changed: 63 additions & 131 deletions

File tree

‎pipelines/cpCore.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,28 +44,28 @@ tasks:
4444
atools.biasMeanByDate: CalibAmpScatterTool
4545
atools.biasMeanByDate.prep.panelKey: amplifier
4646
atools.biasMeanByDate.prep.dataKey: mjd
47-
atools.biasMeanByDate.prep.quantityKey: BIAS_MEAN
47+
atools.biasMeanByDate.prep.quantityKey: ["BIAS_MEAN"]
4848
atools.biasMeanByDate.produce.plot.xAxisLabel: "MJD"
4949
atools.biasMeanByDate.produce.plot.yAxisLabel: "Residual bias mean (ADU)"
5050

5151
atools.biasStdByDate: CalibAmpScatterTool
5252
atools.biasStdByDate.prep.panelKey: amplifier
5353
atools.biasStdByDate.prep.dataKey: mjd
54-
atools.biasStdByDate.prep.quantityKey: BIAS_NOISE
54+
atools.biasStdByDate.prep.quantityKey: ["BIAS_NOISE"]
5555
atools.biasStdByDate.produce.plot.xAxisLabel: "MJD"
5656
atools.biasStdByDate.produce.plot.yAxisLabel: "Residual bias stdev (ADU)"
5757

5858
atools.biasROCornerByDate: CalibAmpScatterTool
5959
atools.biasROCornerByDate.prep.panelKey: amplifier
6060
atools.biasROCornerByDate.prep.dataKey: mjd
61-
atools.biasROCornerByDate.prep.quantityKey: BIAS_AMP_CORNER
61+
atools.biasROCornerByDate.prep.quantityKey: ["BIAS_AMP_CORNER"]
6262
atools.biasROCornerByDate.produce.plot.xAxisLabel: "MJD"
6363
atools.biasROCornerByDate.produce.plot.yAxisLabel: "Residual bias mean at RO corner (ADU)"
6464

6565
atools.biasTestsByDate: CalibAmpScatterTool
6666
atools.biasTestsByDate.prep.panelKey: amplifier
6767
atools.biasTestsByDate.prep.dataKey: mjd
68-
atools.biasTestsByDate.prep.quantityKey: BIAS_VERIFY_MEAN
68+
atools.biasTestsByDate.prep.quantityKey: ["BIAS_VERIFY_MEAN"]
6969
atools.biasTestsByDate.produce.plot.xAxisLabel: "MJD"
7070
atools.biasTestsByDate.produce.plot.yAxisLabel: "Bias Test Passing"
7171

@@ -118,21 +118,21 @@ tasks:
118118
atools.darkMeanByDate: CalibAmpScatterTool
119119
atools.darkMeanByDate.prep.panelKey: amplifier
120120
atools.darkMeanByDate.prep.dataKey: mjd
121-
atools.darkMeanByDate.prep.quantityKey: DARK_MEAN
121+
atools.darkMeanByDate.prep.quantityKey: ["DARK_MEAN"]
122122
atools.darkMeanByDate.produce.plot.xAxisLabel: "MJD"
123123
atools.darkMeanByDate.produce.plot.yAxisLabel: "Residual dark mean (ADU)"
124124

125125
atools.darkStdByDate: CalibAmpScatterTool
126126
atools.darkStdByDate.prep.panelKey: amplifier
127127
atools.darkStdByDate.prep.dataKey: mjd
128-
atools.darkStdByDate.prep.quantityKey: DARK_NOISE
128+
atools.darkStdByDate.prep.quantityKey: ["DARK_NOISE"]
129129
atools.darkStdByDate.produce.plot.xAxisLabel: "MJD"
130130
atools.darkStdByDate.produce.plot.yAxisLabel: "Residual dark std (ADU)"
131131

132132
atools.darkTestsByDate: CalibAmpScatterTool
133133
atools.darkTestsByDate.prep.panelKey: amplifier
134134
atools.darkTestsByDate.prep.dataKey: mjd
135-
atools.darkTestsByDate.prep.quantityKey: DARK_VERIFY_MEAN
135+
atools.darkTestsByDate.prep.quantityKey: ["DARK_VERIFY_MEAN"]
136136
atools.darkTestsByDate.produce.plot.xAxisLabel: "MJD"
137137
atools.darkTestsByDate.produce.plot.yAxisLabel: "Dark Test Passing"
138138

@@ -148,7 +148,7 @@ tasks:
148148
atools.flatTestsByDate: CalibAmpScatterTool
149149
atools.flatTestsByDate.prep.panelKey: amplifier
150150
atools.flatTestsByDate.prep.dataKey: mjd
151-
atools.flatTestsByDate.prep.quantityKey: FLAT_VERIFY_NOISE
151+
atools.flatTestsByDate.prep.quantityKey: ["FLAT_VERIFY_NOISE"]
152152
# TODO: DM-43878
153153
# FLAT_DET_VERIFY_SCATTER
154154
atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD"
@@ -210,7 +210,7 @@ tasks:
210210
atools.ptcPlot: CalibAmpScatterTool
211211
atools.ptcPlot.prep.panelKey: amplifier
212212
atools.ptcPlot.prep.dataKey: PTC_PTC_RAW_MEANS
213-
atools.ptcPlot.prep.quantityKey: PTC_PTC_RAW_VARIANCE
213+
atools.ptcPlot.prep.quantityKey: ["PTC_PTC_RAW_VARIANCE"]
214214
atools.ptcPlot.produce.plot.xAxisLabel: "Exposure Pair Flux (ADU)"
215215
atools.ptcPlot.produce.plot.yAxisLabel: "Exposure Pair Variance (ADU^2)"
216216
python: |

‎python/lsst/analysis/tools/actions/plot/gridPlot.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,7 @@ class GridPlot(PlotAction):
8484
doc="Independent data definitions. The key of this dict is the panel ID. The values are keys of data "
8585
"to plot (comma-separated for multiple) where each key may be a subset of a full key.",
8686
)
87-
figsize = ListField[float](
88-
doc="Figure size.",
89-
default=[8, 8],
90-
)
87+
figsize = ListField[float](doc="Figure size.", default=[8, 8], length=2)
9188
dpi = Field[float](
9289
doc="Dots per inch.",
9390
default=150,

‎python/lsst/analysis/tools/atools/calibQuantityProfile.py‎

Lines changed: 46 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
# along with this program. If not, see <https://www.gnu.org/licenses/>.
2121
from __future__ import annotations
2222

23-
from lsst.analysis.tools.interfaces._interfaces import KeyedDataSchema
24-
2523
__all__ = (
2624
"CalibQuantityBaseTool",
2725
"CalibQuantityAmpProfileScatterTool",
@@ -31,14 +29,34 @@
3129
"CalibPtcCovarScatterTool",
3230
)
3331

34-
from typing import cast
32+
from typing import Optional
3533

36-
from lsst.pex.config import Field
34+
import numpy as np
35+
from lsst.pex.config import Field, ListField
3736
from lsst.pex.config.configurableActions import ConfigurableActionField
3837

3938
from ..actions.plot.elements import HistElement, ScatterElement
4039
from ..actions.plot.gridPlot import GridPanelConfig, GridPlot
41-
from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, PlotElement, Vector
40+
from ..interfaces import AnalysisTool, KeyedData, KeyedDataAction, KeyedDataSchema, PlotElement, Vector
41+
42+
_CALIB_AMP_NAME_DICT: dict[int, str] = {
43+
0: "C00",
44+
1: "C01",
45+
2: "C02",
46+
3: "C03",
47+
4: "C04",
48+
5: "C05",
49+
6: "C06",
50+
7: "C07",
51+
8: "C10",
52+
9: "C11",
53+
10: "C12",
54+
11: "C13",
55+
12: "C14",
56+
13: "C15",
57+
14: "C16",
58+
15: "C17",
59+
}
4260

4361

4462
class PrepRepacker(KeyedDataAction):
@@ -50,61 +68,23 @@ class PrepRepacker(KeyedDataAction):
5068
dataKey = Field[str](
5169
doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
5270
)
53-
quantityKey = Field[str](
54-
doc="Quantity selector. The actual data quantities to be plotted.",
71+
quantityKey = ListField[str](
72+
doc="Quantity selector. The actual data quantities to be plotted.", minLength=1, optional=False
5573
)
5674

5775
def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
58-
repackedData = {}
59-
# Loop over the length of the data vector and repack row by row
60-
for i in range(len(cast(Vector, data[self.panelKey]))):
61-
panelVec = cast(Vector, data[self.panelKey])
62-
dataVec = cast(Vector, data[self.dataKey])
63-
quantityVec = cast(Vector, data[self.quantityKey])
64-
repackedData[f"{panelVec[i]}_{dataVec[i]}_{self.quantityKey}"] = quantityVec[i]
65-
return repackedData
66-
67-
def getInputSchema(self) -> KeyedDataSchema:
68-
return (
69-
(self.panelKey, Vector),
70-
(self.dataKey, Vector),
71-
(self.quantityKey, Vector),
72-
)
73-
74-
def addInputSchema(self, inputSchema: KeyedDataSchema) -> None:
75-
pass
76-
77-
78-
class SingleValueRepacker(KeyedDataAction):
79-
"""Prep action to repack data."""
80-
81-
panelKey = Field[str](
82-
doc="Panel selector. Data will be separated into multiple panels based on this key.",
83-
)
84-
dataKey = Field[str](
85-
doc="Data selector. Data will be separated into multiple groups in a single panel based on this key.",
86-
)
87-
quantityKey = Field[str](
88-
doc="Quantity selector. The actual data quantities to be plotted.",
89-
)
90-
91-
def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
92-
repackedData = {}
76+
repackedData: dict[str, Vector] = {}
9377
uniquePanelKeys = list(set(data[self.panelKey]))
9478

95-
# Loop over data vector to repack information as it is expected.
96-
for i in range(len(uniquePanelKeys)):
97-
repackedData[f"{uniquePanelKeys[i]}_x"] = []
98-
repackedData[f"{uniquePanelKeys[i]}"] = []
99-
100-
panelVec = cast(Vector, data[self.panelKey])
101-
dataVec = cast(Vector, data[self.dataKey])
102-
quantityVec = cast(Vector, data[self.quantityKey])
103-
104-
for i in range(len(panelVec)):
105-
repackedData[f"{panelVec[i]}_x"].append(dataVec[i])
106-
repackedData[f"{panelVec[i]}"].append(quantityVec[i])
79+
for pKey in uniquePanelKeys:
80+
# Make a boolean array that selects the correct panel data
81+
sel: np.ndarray = data[self.panelKey] == pKey
10782

83+
# Setup the x axis
84+
repackedData[f"{pKey}_x"] = data[self.dataKey][sel]
85+
for qkey in self.quantityKey:
86+
# Setup a y axis series for each quantityKey
87+
repackedData[f"{pKey}_{qkey}"] = data[qkey][sel]
10888
return repackedData
10989

11090
def getInputSchema(self) -> KeyedDataSchema:
@@ -136,6 +116,12 @@ class CalibQuantityBaseTool(AnalysisTool):
136116
doc="Plot element.",
137117
)
138118

119+
def _get_xKey_dict(self, yKeys: Optional[dict[int, str]] = None) -> dict[int, str]:
120+
"""Generate the dictionary of x axis keys from the y axis ones"""
121+
if yKeys is None:
122+
yKeys = self.produce.plot.valsGroupBy
123+
return {k: f"{v}_x" for k, v in yKeys.items()}
124+
139125
def setDefaults(self):
140126
super().setDefaults()
141127

@@ -152,24 +138,7 @@ def setDefaults(self):
152138
self.produce.plot.numCols = 4
153139

154140
# Values to group by to distinguish between data in differing panels
155-
self.produce.plot.valsGroupBy = {
156-
0: "C00",
157-
1: "C01",
158-
2: "C02",
159-
3: "C03",
160-
4: "C04",
161-
5: "C05",
162-
6: "C06",
163-
7: "C07",
164-
8: "C10",
165-
9: "C11",
166-
10: "C12",
167-
11: "C13",
168-
12: "C14",
169-
13: "C15",
170-
14: "C16",
171-
15: "C17",
172-
}
141+
self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT
173142

174143
def finalize(self):
175144
super().finalize()
@@ -206,52 +175,17 @@ def setDefaults(self):
206175
self.plotElement = ScatterElement()
207176

208177
# Repack the input data into a usable format
209-
self.prep = SingleValueRepacker()
178+
self.prep = PrepRepacker()
210179

211180
self.produce.plot = GridPlot()
212181
self.produce.plot.panels = {}
213182
self.produce.plot.numRows = 4
214183
self.produce.plot.numCols = 4
215184

216185
# Values to use for x-axis data
217-
self.produce.plot.xDataKeys = {
218-
0: "C00_x",
219-
1: "C01_x",
220-
2: "C02_x",
221-
3: "C03_x",
222-
4: "C04_x",
223-
5: "C05_x",
224-
6: "C06_x",
225-
7: "C07_x",
226-
8: "C10_x",
227-
9: "C11_x",
228-
10: "C12_x",
229-
11: "C13_x",
230-
12: "C14_x",
231-
13: "C15_x",
232-
14: "C16_x",
233-
15: "C17_x",
234-
}
235-
236186
# Values to group by to distinguish between data in differing panels
237-
self.produce.plot.valsGroupBy = {
238-
0: "C00",
239-
1: "C01",
240-
2: "C02",
241-
3: "C03",
242-
4: "C04",
243-
5: "C05",
244-
6: "C06",
245-
7: "C07",
246-
8: "C10",
247-
9: "C11",
248-
10: "C12",
249-
11: "C13",
250-
12: "C14",
251-
13: "C15",
252-
14: "C16",
253-
15: "C17",
254-
}
187+
self.produce.plot.valsGroupBy = _CALIB_AMP_NAME_DICT
188+
self.produce.plot.xDataKeys = self._get_xKey_dict()
255189

256190
self.prep.panelKey = "amplifier"
257191
self.prep.dataKey = "mjd"
@@ -277,7 +211,7 @@ def setDefaults(self):
277211
self.plotElement = ScatterElement()
278212

279213
# Repack the input data into a usable format
280-
self.prep = SingleValueRepacker()
214+
self.prep = PrepRepacker()
281215

282216
self.produce.plot = GridPlot()
283217
self.produce.plot.panels = {}
@@ -287,10 +221,7 @@ def setDefaults(self):
287221
# Values to group by to distinguish between data in differing panels
288222
self.produce.plot.valsGroupBy = {0: "bank1", 1: "bank0"}
289223

290-
self.produce.plot.xDataKeys = {
291-
0: "bank1_x",
292-
1: "bank0_x",
293-
}
224+
self.produce.plot.xDataKeys = self._get_xKey_dict()
294225
self.prep.panelKey = "amplifier"
295226
self.prep.dataKey = "mjd"
296227

‎python/lsst/analysis/tools/interfaces/_actions.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
import warnings
4141
from abc import abstractmethod
4242
from dataclasses import dataclass
43-
from typing import TYPE_CHECKING, Iterable
43+
from typing import TYPE_CHECKING, Any, Generator, GenericAlias, Iterable, Sequence
4444

4545
import lsst.pex.config as pexConfig
4646
from lsst.pex.config.configurableActions import ConfigurableAction, ConfigurableActionField
@@ -108,7 +108,7 @@ def getOutputSchema(self) -> KeyedDataSchema | None:
108108
"""
109109
return None
110110

111-
def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema:
111+
def getFormattedInputSchema(self, **kwargs) -> Generator[tuple[Any, GenericAlias], None, None]:
112112
"""Return input schema, with keys formatted with any arguments supplied
113113
by kwargs passed to this method.
114114
@@ -119,7 +119,11 @@ def getFormattedInputSchema(self, **kwargs) -> KeyedDataSchema:
119119
action, formatted with any input arguments (e.g. band='i')
120120
"""
121121
for key, typ in self.getInputSchema():
122-
yield key.format_map(kwargs), typ
122+
if isinstance(key, Sequence) and not isinstance(key, str):
123+
for subkey in key:
124+
yield subkey.format_map(kwargs), typ
125+
else:
126+
yield key.format_map(kwargs), typ
123127

124128
def addInputSchema(self, inputSchema: KeyedDataSchema) -> None:
125129
"""Add the supplied inputSchema argument to the class such that it will

0 commit comments

Comments
 (0)