diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e46faaa..d0b4c3f91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,6 +340,42 @@ jobs: chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} + macos-s2n: + runs-on: macos-14 # latest + env: + AWS_CRT_USE_NON_FIPS_TLS_13: 1 + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + macos-x64-s2n: + runs-on: macos-14-large # latest + env: + AWS_CRT_USE_NON_FIPS_TLS_13: 1 + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + openbsd: runs-on: ubuntu-24.04 # latest strategy: diff --git a/README.md b/README.md index 64307869d..e3c5d7972 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,33 @@ If you **must** use fork with aws-crt-python, you may be able to avoid hangs and For an example, see `test.test_s3.py.S3RequestTest.test_fork_workaround` . -## Mac-Only TLS Behavior +## macOS TLS Configuration -Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level: +By default on macOS, aws-crt-python uses Apple Secure Transport for TLS. This provides FIPS-compliant cryptography +and integration with the macOS Keychain (e.g. PKCS#12 credentials), but is limited to TLS 1.2. + +To enable TLS 1.3 on macOS, set the environment variable: + +``` +export AWS_CRT_USE_NON_FIPS_TLS_13=1 +``` + +This switches the TLS backend from Apple Secure Transport to [s2n-tls](https://github.com/aws/s2n-tls) with +[aws-lc](https://github.com/aws/aws-lc) as the underlying libcrypto. The tradeoffs are: + +| | Secure Transport (default) | s2n-tls (`AWS_CRT_USE_NON_FIPS_TLS_13=1`) | +|---|---|---| +| TLS versions | Up to TLS 1.2 | Up to TLS 1.3 | +| FIPS compliance | Yes | No | +| macOS Keychain integration | Yes (PKCS#12, system certs) | No | + +This variable is checked at runtime and only affects macOS. It has no effect on Linux (which always uses s2n-tls) +or Windows (which always uses Schannel). Both TLS backends are compiled into the binary when building on macOS; +the environment variable selects which one is used. + +### Keychain Behavior + +Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level: ``` static: certificate has an existing certificate-key pair that was previously imported into the Keychain. Using key from Keychain instead of the one provided. @@ -110,8 +134,9 @@ You can enable the crash handler by setting the environment variable `AWS_CRT_CR ### OpenSSL and LibCrypto aws-crt-python does not use OpenSSL for TLS. -On Apple and Windows devices, the OS's default TLS library is used. -On Unix devices, [s2n-tls](https://github.com/aws/s2n-tls) is used. +On Windows, the OS's default TLS library (Schannel) is used. +On Apple (macOS), both Secure Transport and s2n-tls are compiled in; the backend is selected at runtime (see [macOS TLS Configuration](#macos-tls-configuration) below). +On other Unix devices, [s2n-tls](https://github.com/aws/s2n-tls) is used. But s2n-tls uses libcrypto, the cryptography math library bundled with OpenSSL. To simplify installation, aws-crt-python has its own copy of libcrypto. diff --git a/crt/CMakeLists.txt b/crt/CMakeLists.txt index 965f110e1..9e32b5af2 100644 --- a/crt/CMakeLists.txt +++ b/crt/CMakeLists.txt @@ -32,9 +32,10 @@ option(AWS_USE_LIBCRYPTO_TO_SUPPORT_ED25519_EVERYWHERE "Set this if you want to string(REPLACE "-g" "-g1" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") string(REPLACE "-g" "-g1" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") -# On Unix we use S2N for TLS and AWS-LC crypto. -# (On Windows and Apple we use the default OS libraries) -if ((UNIX AND NOT APPLE) OR AWS_USE_LIBCRYPTO_TO_SUPPORT_ED25519_EVERYWHERE) +# On Linux and BSD, we use S2N for TLS and AWS-LC crypto. +# On Windows, we use the default OS libraries. +# On Apple, we use the default OS libraries by default, but support S2N usage. +if (UNIX OR AWS_USE_LIBCRYPTO_TO_SUPPORT_ED25519_EVERYWHERE) option(USE_OPENSSL "Set this if you want to use your system's OpenSSL compatible libcrypto" OFF) include(AwsPrebuildDependency) @@ -48,7 +49,7 @@ if ((UNIX AND NOT APPLE) OR AWS_USE_LIBCRYPTO_TO_SUPPORT_ED25519_EVERYWHERE) -DCMAKE_BUILD_TYPE=RelWithDebInfo # Use the same build type as the rest of the project ) - if (APPLE OR WIN32) + if (WIN32) # Libcrypto implementations typically have several chunky pregenerated tables that add a lot # to artifact size. We dont really need them for ed25519 case on win/mac, so favor # smaller binary over perf here. @@ -72,7 +73,10 @@ if ((UNIX AND NOT APPLE) OR AWS_USE_LIBCRYPTO_TO_SUPPORT_ED25519_EVERYWHERE) endif() -if(UNIX AND NOT APPLE) +# Build s2n-tls on all Unix platforms (Linux, BSD, macOS). +# On macOS (Darwin), both Secure Transport and s2n are built; the TLS backend +# is selected at runtime via the AWS_CRT_USE_NON_FIPS_TLS_13 environment variable. +if(UNIX) # prebuild s2n-tls. aws_prebuild_dependency( DEPENDENCY_NAME S2N @@ -80,6 +84,11 @@ if(UNIX AND NOT APPLE) CMAKE_ARGUMENTS -DUNSAFE_TREAT_WARNINGS_AS_ERRORS=OFF -DBUILD_TESTING=OFF + # On Intel Macs, Homebrew installs to /usr/local, which is in the default + # system header search path. Without this flag, s2n picks up Homebrew's OpenSSL + # headers instead of the bundled aws-lc headers. Not needed on ARM where Homebrew + # uses /opt/homebrew (not in default search paths), but harmless to set everywhere. + -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON ) endif() diff --git a/crt/aws-c-io b/crt/aws-c-io index 1ec8081f2..bdfd85e29 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 1ec8081f208ef8d51381889eda3bda9756fd5bb5 +Subproject commit bdfd85e293965439146bc30a70379e7889c27bb7 diff --git a/setup.py b/setup.py index 380e5e948..032bff830 100644 --- a/setup.py +++ b/setup.py @@ -301,7 +301,7 @@ def __init__(self, name, extra_cmake_args=[], libname=None): # aws-lc produces libcrypto.a AWS_LIBS.append(AwsLib('aws-lc', libname='crypto')) -if sys.platform != 'darwin' and sys.platform != 'win32': +if sys.platform != 'win32': AWS_LIBS.append(AwsLib('s2n')) AWS_LIBS.append(AwsLib('aws-c-common')) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 15e1e7d0a..1e6dbd080 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -5,6 +5,7 @@ from awscrt import mqtt5, io, http, exceptions from test import test_retry_wrapper, NativeResourceTest import os +import sys import unittest import uuid import time @@ -303,6 +304,38 @@ def _test_direct_connect_mutual_tls(self): def test_direct_connect_mutual_tls(self): test_retry_wrapper(self._test_direct_connect_mutual_tls) + def _test_direct_connect_mutual_tls13(self): + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_TLS13_HOST") + input_cert = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_CERT") + input_key = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_KEY") + + client_options = mqtt5.ClientOptions( + host_name=input_host_name, + port=8883 + ) + tls_ctx_options = io.TlsContextOptions.create_client_with_mtls_from_path( + input_cert, + input_key + ) + client_options.tls_ctx = io.ClientTlsContext(tls_ctx_options) + + callbacks = Mqtt5TestCallbacks() + client = self._create_client(client_options=client_options, callbacks=callbacks) + client.start() + + # On macOS with Secure Transport (the default), TLS 1.3 is not supported, + # so the connection to a TLS-1.3-only host must fail. + if sys.platform == 'darwin' and not os.environ.get('AWS_CRT_USE_NON_FIPS_TLS_13'): + callbacks.future_connection_failure.result(TIMEOUT) + else: + callbacks.future_connection_success.result(TIMEOUT) + + client.stop() + callbacks.future_stopped.result(TIMEOUT) + + def test_direct_connect_mutual_tls13(self): + test_retry_wrapper(self._test_direct_connect_mutual_tls13) + def _test_direct_connect_http_proxy_tls(self): input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_TLS_HOST") input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_TLS_PORT")) diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index 96057834e..2354c944b 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -130,6 +130,9 @@ def _test_mqtt5_cred_pkcs12(self): client.stop() callbacks.future_stopped.result(TIMEOUT) + # When AWS_CRT_USE_NON_FIPS_TLS_13 is set, the TLS backend on macOS switches from + # Secure Transport to s2n-tls, which doesn't support PKCS#12. + @unittest.skipIf(os.environ.get('AWS_CRT_USE_NON_FIPS_TLS_13'), "PKCS12 not supported with non-FIPS TLS 1.3") def test_mqtt5_cred_pkcs12(self): test_retry_wrapper(self._test_mqtt5_cred_pkcs12) diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 58b99b4ce..832c639d8 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -46,6 +46,9 @@ def _test_mqtt311_cred_pkcs12(self): connection.connect().result(TIMEOUT) connection.disconnect().result(TIMEOUT) + # When AWS_CRT_USE_NON_FIPS_TLS_13 is set, the TLS backend on macOS switches from + # Secure Transport to s2n-tls, which doesn't support PKCS#12. + @unittest.skipIf(os.environ.get('AWS_CRT_USE_NON_FIPS_TLS_13'), "PKCS12 not supported with non-FIPS TLS 1.3") def test_mqtt311_cred_pkcs12(self): test_retry_wrapper(self._test_mqtt311_cred_pkcs12)