Merge pull request #220 from lsst/tickets/DM-32437

DM-32437: format with black and isort

Author: TallJimbo

.github/workflows/formatting.yaml

@@ -0,0 +1,27 @@
+name: check Python formatting
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install isort and black
+        run: pip install isort black
+
+      - name: Run isort
+        run: isort --check-only python/ tests/
+
+      - name: Run black
+        run: black --check --verbose --diff python/ tests/

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 21.11b1
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.8
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)

pyproject.toml

@@ -36,3 +36,11 @@
         directory = "removal"
         name = "An API Removal or Deprecation"
         showcontent = false
+
+[tool.black]
+line-length = 110
+target-version = ["py38"]
+
+[tool.isort]
+profile = "black"
+line_length = 110

python/lsst/__init__.py

@@ -1,3 +1,5 @@
 import pkgutil
+
 import lsstimport
+
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/pipe/__init__.py

@@ -1,3 +1,5 @@
 import pkgutil
+
 import lsstimport
+
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/pipe/base/__init__.py

@@ -1,20 +1,19 @@
-from .version import *
+from . import connectionTypes, pipelineIR
+from ._status import *
+from ._task_metadata import *
 from .argumentParser import *
-from .struct import *
-from .task import *
+from .butlerQuantumContext import *
 from .cmdLineTask import *
-from .timer import *
 from .config import *
-from .pipelineTask import *
-from .pipeline import *
+from .connections import *
+from .executionButlerBuilder import *
 from .graph import *
 from .graphBuilder import *
-from .taskFactory import *
+from .pipeline import *
+from .pipelineTask import *
+from .struct import *
+from .task import *
 from .task_logging import getTaskLogger
-from .connections import *
-from .butlerQuantumContext import *
-from . import connectionTypes
-from . import pipelineIR
-from .executionButlerBuilder import *
-from ._status import *
-from ._task_metadata import *
+from .taskFactory import *
+from .timer import *
+from .version import *

python/lsst/pipe/base/_datasetQueryConstraints.py

@@ -26,9 +26,8 @@
 
 __all__ = ("DatasetQueryConstraintVariant",)
 
-from typing import Type, Protocol, Iterable, Iterator
-
 import warnings
+from typing import Iterable, Iterator, Protocol, Type
 
 
 class DatasetQueryConstraintVariant(Iterable, Protocol):
@@ -54,9 +53,10 @@ class DatasetQueryConstraintVariant(Iterable, Protocol):
     Variants can be directly used, or automatically be selected by using the
     `fromExpression` class method given a valid string.
     """
-    ALL: 'Type[_ALL]'
-    OFF: 'Type[_OFF]'
-    LIST: 'Type[_LIST]'
+
+    ALL: "Type[_ALL]"
+    OFF: "Type[_OFF]"
+    LIST: "Type[_LIST]"
 
     @classmethod
     def __subclasshook__(cls, subclass):
@@ -65,7 +65,7 @@ def __subclasshook__(cls, subclass):
         return False
 
     @classmethod
-    def fromExpression(cls, expression: str) -> 'DatasetQueryConstraintVariant':
+    def fromExpression(cls, expression: str) -> "DatasetQueryConstraintVariant":
         """Select and return the correct Variant that corresponds to the input
         expression.
 
@@ -75,15 +75,15 @@ def fromExpression(cls, expression: str) -> 'DatasetQueryConstraintVariant':
         """
         if not isinstance(expression, str):
             raise ValueError("Expression must be a string")
-        elif expression == 'all':
+        elif expression == "all":
             return cls.ALL
-        elif expression == 'off':
+        elif expression == "off":
             return cls.OFF
         else:
             if " " in expression:
                 warnings.warn("Witespace found in expression will be trimmed", RuntimeWarning)
-                expression = expression.replace(' ', '')
-            members = expression.split(',')
+                expression = expression.replace(" ", "")
+            members = expression.split(",")
             return cls.LIST(members)
 
 
@@ -143,8 +143,10 @@ def __eq__(self, o: object) -> bool:
 
 
 def suppressInit(self):
-    raise NotImplementedError("DatasetQueryConstraintVariants cannot be directly instantiated. "
-                              "Please use the variants or the fromExpression class method")
+    raise NotImplementedError(
+        "DatasetQueryConstraintVariants cannot be directly instantiated. "
+        "Please use the variants or the fromExpression class method"
+    )
 
 
 DatasetQueryConstraintVariant.__init__ = suppressInit

python/lsst/pipe/base/_task_metadata.py

@@ -21,14 +21,14 @@
 
 __all__ = ["TaskMetadata"]
 
-import numbers
 import itertools
+import numbers
 import warnings
 from collections.abc import Sequence
-from deprecated.sphinx import deprecated
+from typing import Any, Collection, Dict, List, Mapping, Protocol, Union
 
-from typing import Dict, List, Union, Any, Mapping, Protocol, Collection
-from pydantic import BaseModel, StrictInt, StrictFloat, StrictBool, StrictStr, Field
+from deprecated.sphinx import deprecated
+from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr
 
 _DEPRECATION_REASON = "Will be removed after v25."
 _DEPRECATION_VERSION = "v24"
@@ -73,8 +73,9 @@ class TaskMetadata(BaseModel):
     """
 
     scalars: Dict[str, Union[StrictFloat, StrictInt, StrictBool, StrictStr]] = Field(default_factory=dict)
-    arrays: Dict[str, Union[List[StrictFloat], List[StrictInt], List[StrictBool],
-                            List[StrictStr]]] = Field(default_factory=dict)
+    arrays: Dict[str, Union[List[StrictFloat], List[StrictInt], List[StrictBool], List[StrictStr]]] = Field(
+        default_factory=dict
+    )
     metadata: Dict[str, "TaskMetadata"] = Field(default_factory=dict)
 
     @classmethod
@@ -175,8 +176,11 @@ def add(self, name, value):
 
         self.metadata[key0].add(".".join(keys), value)
 
-    @deprecated(reason="Cast the return value to float explicitly. " + _DEPRECATION_REASON,
-                version=_DEPRECATION_VERSION, category=FutureWarning)
+    @deprecated(
+        reason="Cast the return value to float explicitly. " + _DEPRECATION_REASON,
+        version=_DEPRECATION_VERSION,
+        category=FutureWarning,
+    )
     def getAsDouble(self, key):
         """Return the value cast to a `float`.
 
@@ -288,7 +292,7 @@ def names(self, topLevelOnly: bool = True):
             for k, v in self.items():
                 names.add(k)  # Always include the current level
                 if isinstance(v, TaskMetadata):
-                    names.update({k + '.' + item for item in v.names(topLevelOnly=topLevelOnly)})
+                    names.update({k + "." + item for item in v.names(topLevelOnly=topLevelOnly)})
             return names
 
     def paramNames(self, topLevelOnly):
@@ -318,14 +322,20 @@ def paramNames(self, topLevelOnly):
                 paramNames.add(k)
         return paramNames
 
-    @deprecated(reason="Use standard assignment syntax. " + _DEPRECATION_REASON,
-                version=_DEPRECATION_VERSION, category=FutureWarning)
+    @deprecated(
+        reason="Use standard assignment syntax. " + _DEPRECATION_REASON,
+        version=_DEPRECATION_VERSION,
+        category=FutureWarning,
+    )
     def set(self, key, item):
         """Set the value of the supplied key."""
         self.__setitem__(key, item)
 
-    @deprecated(reason="Use standard del dict syntax. " + _DEPRECATION_REASON,
-                version=_DEPRECATION_VERSION, category=FutureWarning)
+    @deprecated(
+        reason="Use standard del dict syntax. " + _DEPRECATION_REASON,
+        version=_DEPRECATION_VERSION,
+        category=FutureWarning,
+    )
     def remove(self, key):
         """Remove the item without raising if absent."""
         try:
@@ -354,7 +364,7 @@ def _getKeys(key):
             Raised if the key is not a string.
         """
         try:
-            keys = key.split('.')
+            keys = key.split(".")
         except Exception:
             raise KeyError(f"Invalid key '{key}': only string keys are allowed") from None
         return keys
@@ -518,8 +528,10 @@ def _validate_value(self, value):
             type0 = type(value[0])
             for i in value:
                 if type(i) != type0:
-                    raise ValueError("Type mismatch in supplied list. TaskMetadata requires all"
-                                     f" elements have same type but see {type(i)} and {type0}.")
+                    raise ValueError(
+                        "Type mismatch in supplied list. TaskMetadata requires all"
+                        f" elements have same type but see {type(i)} and {type0}."
+                    )
 
             if type0 not in _ALLOWED_PRIMITIVE_TYPES:
                 # Must check to see if we got numpy floats or something.
@@ -528,8 +540,10 @@ def _validate_value(self, value):
                 elif isinstance(value[0], numbers.Real):
                     type_cast = float
                 else:
-                    raise ValueError(f"Supplied list has element of type '{type0}'. "
-                                     "TaskMetadata can only accept primitive types in lists.")
+                    raise ValueError(
+                        f"Supplied list has element of type '{type0}'. "
+                        "TaskMetadata can only accept primitive types in lists."
+                    )
 
                 value = [type_cast(v) for v in value]