diff --git a/lemur/acme_providers/__init__.py b/lemur/acme_providers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py deleted file mode 100644 index d0a6c06bcb..0000000000 --- a/lemur/acme_providers/cli.py +++ /dev/null @@ -1,205 +0,0 @@ -import json -import time - -import arrow -import click -from flask import current_app -from flask.cli import with_appcontext -from sentry_sdk import capture_exception - -from lemur.common.utils import check_validation -from lemur.constants import SUCCESS_METRIC_STATUS -from lemur.plugins import plugins -from lemur.plugins.lemur_acme.plugin import AcmeHandler -from lemur.plugins.lemur_aws import s3 - - -@click.group(name="acme", help="Handles all ACME related tasks") -@with_appcontext -def cli(): - pass - - -@cli.command("dnstest") -@click.option( - "-d", - "--domain", - "domain", - required=True, - help="Name of the Domain to store to (ex. \"_acme-chall.test.com\"." -) -@click.option( - "-t", - "--token", - "token", - required=True, - help="Value of the Token to store in DNS as content." -) -def dnstest_command(domain, token): - dnstest(domain, token) - - -def dnstest(domain, token): - """ - Create, verify, and delete DNS TXT records using an autodetected provider. - """ - click.echo("[+] Starting ACME Tests.") - change_id = (domain, token) - - acme_handler = AcmeHandler() - acme_handler.autodetect_dns_providers(domain) - if not acme_handler.dns_providers_for_domain[domain]: - raise Exception(f"No DNS providers found for domain: {format(domain)}.") - - # Create TXT Records - for dns_provider in acme_handler.dns_providers_for_domain[domain]: - dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - - click.echo(f"[+] Creating TXT Record in `{dns_provider.name}` provider") - change_id = dns_provider_plugin.create_txt_record(domain, token, account_number) - - click.echo("[+] Verifying TXT Record has propagated to DNS.") - click.echo("[+] This step could take a while...") - time.sleep(10) - - # Verify TXT Records - for dns_provider in acme_handler.dns_providers_for_domain[domain]: - dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - - try: - dns_provider_plugin.wait_for_dns_change(change_id, account_number) - click.echo(f"[+] Verified TXT Record in `{dns_provider.name}` provider") - except Exception: - capture_exception() - current_app.logger.debug( - f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " - f"{account_number}", - exc_info=True, - ) - click.echo(f"[+] Unable to Verify TXT Record in `{dns_provider.name}` provider") - - time.sleep(10) - - # Delete TXT Records - for dns_provider in acme_handler.dns_providers_for_domain[domain]: - dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - - # TODO(csine@: Add Exception Handling - dns_provider_plugin.delete_txt_record(change_id, account_number, domain, token) - click.echo(f"[+] Deleted TXT Record in `{dns_provider.name}` provider") - - status = SUCCESS_METRIC_STATUS - click.echo("[+] Done with ACME Tests.") - - -@cli.command("upload_acme_token_s3") -@click.option( - "-t", - "--token", - "token", - default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), - required=False, - help="Value of the Token to store in DNS as content." -) -@click.option( - "-n", - "--token_name", - "token_name", - default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), - required=False, - help="path" -) -@click.option( - "-p", - "--prefix", - "prefix", - default="test/", - required=False, - help="S3 bucket prefix" -) -@click.option( - "-a", - "--account_number", - "account_number", - required=True, - help="AWS Account" -) -@click.option( - "-b", - "--bucket_name", - "bucket_name", - required=True, - help="Bucket Name", -) -def upload_acme_token_s3_command(token, token_name, prefix, account_number, bucket_name): - upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) - - -def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): - """ - This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. - It mainly serves for testing purposes. - :param token: - :param token_name: - :param prefix: - :param account_number: - :param bucket_name: - :return: - """ - additional_options = [ - { - "name": "bucket", - "value": bucket_name, - "type": "str", - "required": True, - "validation": check_validation(r"[0-9a-z.-]{3,63}"), - "helpMessage": "Must be a valid S3 bucket name!", - }, - { - "name": "accountNumber", - "type": "str", - "value": account_number, - "required": True, - "validation": check_validation(r"[0-9]{12}"), - "helpMessage": "A valid AWS account number with permission to access S3", - }, - { - "name": "region", - "type": "str", - "default": "us-east-1", - "required": False, - "helpMessage": "Region bucket exists", - "available": ["us-east-1", "us-west-2", "eu-west-1"], - }, - { - "name": "encrypt", - "type": "bool", - "value": False, - "required": False, - "helpMessage": "Enable server side encryption", - "default": True, - }, - { - "name": "prefix", - "type": "str", - "value": prefix, - "required": False, - "helpMessage": "Must be a valid S3 object prefix!", - }, - ] - - p = plugins.get("aws-s3") - p.upload_acme_token(token_name, token, additional_options) - - if not prefix.endswith("/"): - prefix + "/" - - token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) - assert (token_res == token) - s3.delete(bucket_name, prefix + token_name, account_number=account_number) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index e210dd9b3b..10f959009f 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -25,20 +25,6 @@ from lemur.certificates.service import upload -def _validate_acme_url_in_options(options_json: str) -> None: - # Imported here, rather than at module scope, to avoid a circular import: - # lemur.plugins.lemur_acme.acme_handlers imports lemur.authorities.service. - from lemur.plugins.lemur_acme.plugin import validate_acme_url - - try: - options = json.loads(options_json) - except (json.JSONDecodeError, TypeError): - return - for option in options: - if isinstance(option, dict) and option.get("name") == "acme_url": - validate_acme_url(option.get("value", "")) - - def update(authority_id, description, owner, active, roles, options: Optional[str] = None): """ Update an authority with new values. @@ -54,7 +40,6 @@ def update(authority_id, description, owner, active, roles, options: Optional[st authority.description = description authority.owner = owner if options: - _validate_acme_url_in_options(options) authority.options = options log_service.audit_log("update_authority", authority.name, "Updating authority") # check ui what can be updated @@ -71,7 +56,6 @@ def update_options(authority_id, options): """ authority = get(authority_id) - _validate_acme_url_in_options(options) authority.options = options return database.update(authority) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 5a5ee3b360..2174dc8620 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -23,7 +23,6 @@ from lemur.certificates import cli as cli_certificate from lemur.certificates import service as certificate_service from lemur.common.redis import RedisHandler -from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.dns_providers import cli as cli_dns_providers from lemur.extensions import metrics from lemur.factory import create_app @@ -311,7 +310,7 @@ def fetch_acme_cert(id, notify_reissue_cert_id=None): error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn - if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS: + if pending_cert.number_attempts > 2: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify diff --git a/lemur/common/utils.py b/lemur/common/utils.py index b75dc817d4..a09758ff91 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -176,16 +176,6 @@ def get_key_type_from_ec_curve(curve_name): ec.SECP384R1().name: "ECCSECP384R1", ec.SECP521R1().name: "ECCSECP521R1", ec.SECP256K1().name: "ECCSECP256K1", - ec.SECT163K1().name: "ECCSECT163K1", - ec.SECT233K1().name: "ECCSECT233K1", - ec.SECT283K1().name: "ECCSECT283K1", - ec.SECT409K1().name: "ECCSECT409K1", - ec.SECT571K1().name: "ECCSECT571K1", - ec.SECT163R2().name: "ECCSECT163R2", - ec.SECT233R1().name: "ECCSECT233R1", - ec.SECT283R1().name: "ECCSECT283R1", - ec.SECT409R1().name: "ECCSECT409R1", - ec.SECT571R1().name: "ECCSECT571R2", } if curve_name in _CURVE_TYPES.keys(): @@ -198,10 +188,8 @@ def generate_private_key(key_type): """ Generates a new private key based on key_type. - Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', - 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', - 'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', - 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2' + Valid key types: RSA2048, RSA4096, ECCPRIME192V1, ECCPRIME256V1, ECCSECP192R1, + ECCSECP224R1, ECCSECP256R1, ECCSECP384R1, ECCSECP521R1, ECCSECP256K1 :param key_type: :return: @@ -216,16 +204,6 @@ def generate_private_key(key_type): "ECCSECP384R1": ec.SECP384R1(), "ECCSECP521R1": ec.SECP521R1(), "ECCSECP256K1": ec.SECP256K1(), - "ECCSECT163K1": ec.SECT163K1(), - "ECCSECT233K1": ec.SECT233K1(), - "ECCSECT283K1": ec.SECT283K1(), - "ECCSECT409K1": ec.SECT409K1(), - "ECCSECT571K1": ec.SECT571K1(), - "ECCSECT163R2": ec.SECT163R2(), - "ECCSECT233R1": ec.SECT233R1(), - "ECCSECT283R1": ec.SECT283R1(), - "ECCSECT409R1": ec.SECT409R1(), - "ECCSECT571R2": ec.SECT571R1(), } if key_type not in CERTIFICATE_KEY_TYPES: diff --git a/lemur/constants.py b/lemur/constants.py index 83f526760f..752beba8b7 100644 --- a/lemur/constants.py +++ b/lemur/constants.py @@ -14,9 +14,6 @@ FAILURE_METRIC_STATUS = "failure" -# when ACME attempts to resolve a certificate try in total 3 times -ACME_ADDITIONAL_ATTEMPTS = 2 - CERTIFICATE_KEY_TYPES = [ "RSA2048", "RSA4096", @@ -28,16 +25,6 @@ "ECCSECP384R1", "ECCSECP521R1", "ECCSECP256K1", - "ECCSECT163K1", - "ECCSECT233K1", - "ECCSECT283K1", - "ECCSECT409K1", - "ECCSECT571K1", - "ECCSECT163R2", - "ECCSECT233R1", - "ECCSECT283R1", - "ECCSECT409R1", - "ECCSECT571R2", ] # For commonly reused regexes used by plugins diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index f6fd4c59bb..c6b04ae46d 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -1,50 +1,8 @@ -import sys - import click from flask.cli import with_appcontext -from sentry_sdk import capture_exception - -from lemur.constants import SUCCESS_METRIC_STATUS -from lemur.dns_providers.service import get_all_dns_providers, set_domains -from lemur.extensions import metrics -from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler -@click.group(name="dns_providers", help="Iterates through all DNS providers and sets DNS zones in the database.") +@click.group(name="dns_providers", help="DNS provider zone management (ACME removed).") @with_appcontext def cli(): pass - - -@cli.command("get_all_zones") -def get_all_zones_command(): - get_all_zones() - - -def get_all_zones(): - """ - Retrieves all DNS providers from the database. Refreshes the zones associated with each DNS provider - """ - click.echo("[+] Starting dns provider zone lookup and configuration.") - dns_providers = get_all_dns_providers() - acme_dns_handler = AcmeDnsHandler() - - function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "", - } - - for dns_provider in dns_providers: - try: - zones = acme_dns_handler.get_all_zones(dns_provider) - set_domains(dns_provider, zones) - except Exception as e: - click.echo(f"[+] Error with DNS Provider {dns_provider.name}: {e}") - log_data["message"] = f"get all zones failed for {dns_provider} {e}." - capture_exception(extra=log_data) - - status = SUCCESS_METRIC_STATUS - - metrics.send("get_all_zones", "counter", 1, metric_tags={"status": status}) - click.echo("[+] Done with dns provider zone lookup and configuration.") diff --git a/lemur/manage.py b/lemur/manage.py index 7b0029b6c1..4d50a96be6 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -16,7 +16,6 @@ from lemur import create_app from lemur import database -from lemur.acme_providers.cli import cli as acme_cli from lemur.api_keys.cli import cli as api_keys_cli from lemur.authorities.models import Authority # noqa from lemur.certificates.cli import cli as certificate_cli @@ -593,7 +592,6 @@ def publish_verisign_units(): def main(): - cli.add_command(acme_cli, "acme") cli.add_command(api_keys_cli, "api_keys") cli.add_command(certificate_cli, "certificate") cli.add_command(dns_provider_cli, "dns_providers") diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 118cbeaff0..6164629a33 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -13,7 +13,6 @@ from flask.cli import with_appcontext from lemur.authorities.service import get as get_authority -from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins @@ -125,7 +124,7 @@ def fetch_all_acme(): error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn - if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS: + if pending_cert.number_attempts > 2: error_log["message"] = "Marking pending certificate as resolved" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify diff --git a/lemur/plugins/lemur_acme/__init__.py b/lemur/plugins/lemur_acme/__init__.py deleted file mode 100644 index e537d47268..0000000000 --- a/lemur/plugins/lemur_acme/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - from importlib.metadata import version - VERSION = version(__name__) -except Exception as e: - VERSION = "unknown" diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py deleted file mode 100644 index 5f5632b8ad..0000000000 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ /dev/null @@ -1,619 +0,0 @@ -""" -.. module: lemur.plugins.lemur_acme.plugin - :platform: Unix - :synopsis: This module contains handlers for certain acme related tasks. It needed to be refactored to avoid circular imports - :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - - Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py - -.. moduleauthor:: Kevin Glisson -.. moduleauthor:: Mikhail Khodorovskiy -.. moduleauthor:: Curtis Castrapel -.. moduleauthor:: Mathias Petermann -""" -import json -import time -from datetime import datetime, timezone, timedelta -from urllib.parse import urlparse - -import OpenSSL.crypto -import dns.resolver -import josepy as jose -import requests -from acme import challenges, errors, messages -from acme.client import ClientV2, ClientNetwork -from acme.errors import TimeoutError -from acme.messages import Error as AcmeError, STATUS_VALID -from certbot import crypto_util as acme_crypto_util -from flask import current_app -from retrying import retry -from sentry_sdk import capture_exception - -from lemur.authorities import service as authorities_service -from lemur.common.utils import data_encrypt, data_decrypt, is_json -from lemur.common.utils import generate_private_key, key_to_alg -from lemur.dns_providers import service as dns_provider_service -from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration -from lemur.extensions import metrics -from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns, nsone - - -class _PinnedClientNetwork(ClientNetwork): - """ClientNetwork that rejects outbound requests to any host other than the ACME directory host. - - Prevents a malicious ACME server from redirecting the client to internal URLs via its - directory/order/authorization/finalize responses (SSRF via server-supplied URLs). - """ - - def __init__(self, *args, pinned_hostname: str, **kwargs): - super().__init__(*args, **kwargs) - self._pinned_hostname = pinned_hostname - - def _send_request(self, method: str, url: str, *args, **kwargs) -> requests.Response: - parsed = urlparse(url) - if parsed.hostname != self._pinned_hostname: - raise InvalidConfiguration( - f"ACME client attempted request to disallowed host {parsed.hostname!r}; " - f"expected {self._pinned_hostname!r}" - ) - return super()._send_request(method, url, *args, **kwargs) - - -class AuthorizationRecord: - def __init__(self, domain, target_domain, authz, dns_challenge, change_id, cname_delegation): - self.domain = domain - self.target_domain = target_domain - self.authz = authz - self.dns_challenge = dns_challenge - self.change_id = change_id - self.cname_delegation = cname_delegation - - -class AcmeHandler: - - def reuse_account(self, authority): - if not authority.options: - raise InvalidAuthority("Invalid authority. Options not set") - existing_key = False - existing_regr = False - - for option in json.loads(authority.options): - if option["name"] == "acme_private_key" and option["value"]: - existing_key = True - if option["name"] == "acme_regr" and option["value"]: - existing_regr = True - - if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"): - existing_key = True - - if not existing_regr and current_app.config.get("ACME_REGR"): - existing_regr = True - - if existing_key and existing_regr: - return True - else: - return False - - def strip_wildcard(self, host): - """Removes the leading wildcard and returns Host and whether it was removed or not (True/False)""" - prefix = "*." - if host.startswith(prefix): - return host[len(prefix):], True - return host, False - - def maybe_add_extension(self, host, dns_provider_options): - if dns_provider_options and dns_provider_options.get( - "acme_challenge_extension" - ): - host = host + dns_provider_options.get("acme_challenge_extension") - return host - - def request_certificate(self, acme_client, authorizations, order): - for authorization in authorizations: - for authz in authorization.authz: - authorization_resource, _ = acme_client.poll(authz) - - deadline = datetime.now() + timedelta(seconds=360) - - try: - orderr = acme_client.poll_authorizations(order, deadline) - orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) - - except (AcmeError, TimeoutError): - capture_exception(extra={"order_url": str(order.uri)}) - metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri}) - current_app.logger.error( - f"Unable to resolve Acme order: {order.uri}", exc_info=True - ) - raise - except errors.ValidationError: - if order.fullchain_pem: - orderr = order - else: - raise - - metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri}) - current_app.logger.info( - f"Successfully resolved Acme order: {order.uri}", exc_info=True - ) - - pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem, - orderr.alternative_fullchains_pem) - - self.log_remaining_validation(orderr.authorizations, acme_client.net.account.uri.replace('https://', '')) - - current_app.logger.debug( - f"{type(pem_certificate)} {type(pem_certificate_chain)}" - ) - return pem_certificate, pem_certificate_chain - - def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, preferred_issuer=None): - - if not preferred_issuer: - preferred_issuer = current_app.config.get("ACME_PREFERRED_ISSUER", None) - if preferred_issuer: - # returns first chain if not match - fullchain_pem = acme_crypto_util.find_chain_with_issuer([fullchain_pem] + alternative_fullchains_pem, - preferred_issuer) - - pem_certificate = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, fullchain_pem - ), - ).decode() - - pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() - - return pem_certificate, pem_certificate_chain - - @retry(stop_max_attempt_number=5, wait_fixed=5000) - def setup_acme_client(self, authority): - return self.setup_acme_client_no_retry(authority) - - def setup_acme_client_no_retry(self, authority): - if not authority.options: - raise InvalidAuthority("Invalid authority. Options not set") - options = {} - - for option in json.loads(authority.options): - options[option["name"]] = option.get("value") - email = options.get("email", current_app.config.get("ACME_EMAIL")) - tel = options.get("telephone", current_app.config.get("ACME_TEL")) - directory_url = options.get( - "acme_url", current_app.config.get("ACME_DIRECTORY_URL") - ) - - existing_key = options.get( - "acme_private_key", current_app.config.get("ACME_PRIVATE_KEY") - ) - existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR")) - - eab_kid = options.get("eab_kid", None) - eab_hmac_key = options.get("eab_hmac_key", None) - - pinned_hostname = urlparse(directory_url).hostname - - if existing_key and existing_regr: - current_app.logger.debug("Reusing existing ACME account") - # Reuse the same account for each certificate issuance - - # existing_key might be encrypted - if not is_json(existing_key): - # decrypt the private key, if not already in plaintext (json format) - existing_key = data_decrypt(existing_key) - - key = jose.JWK.json_loads(existing_key) - regr = messages.RegistrationResource.json_loads(existing_regr) - current_app.logger.debug( - f"Connecting with directory at {directory_url}" - ) - net = _PinnedClientNetwork(key, account=regr, alg=key_to_alg(key), pinned_hostname=pinned_hostname) - directory = ClientV2.get_directory(directory_url, net) - client = ClientV2(directory, net=net) - return client, {} - else: - # Create an account for each certificate issuance - key = jose.JWKRSA(key=generate_private_key("RSA2048")) - - current_app.logger.debug("Creating a new ACME account") - current_app.logger.debug( - f"Connecting with directory at {directory_url}" - ) - - net = _PinnedClientNetwork(key, account=None, timeout=3600, alg=key_to_alg(key), pinned_hostname=pinned_hostname) - directory = ClientV2.get_directory(directory_url, net) - client = ClientV2(directory, net=net) - if eab_kid and eab_hmac_key: - # external account binding (eab_kid and eab_hmac_key could be potentially single use to establish - # long-term credentials) - eab = messages.ExternalAccountBinding.from_data(account_public_key=key.public_key(), - kid=eab_kid, - hmac_key=eab_hmac_key, - directory=client.directory) - registration = client.new_account( - messages.NewRegistration.from_data(email=email, external_account_binding=eab, - terms_of_service_agreed=True) - ) - else: - registration = client.new_account( - messages.NewRegistration.from_data(email=email, terms_of_service_agreed=True) - ) - - # if store_account is checked, add the private_key and registration resources to the options - if options['store_account']: - new_options = json.loads(authority.options) - # the key returned by fields_to_partial_json is missing the key type, so we add it manually - key_dict = key.fields_to_partial_json() - key_dict["kty"] = "RSA" - acme_private_key = { - "name": "acme_private_key", - "value": data_encrypt(json.dumps(key_dict)) - } - new_options.append(acme_private_key) - - acme_regr = { - "name": "acme_regr", - "value": json.dumps({"body": {}, "uri": registration.uri}) - } - new_options.append(acme_regr) - - authorities_service.update_options(authority.id, options=json.dumps(new_options)) - - current_app.logger.debug(f"Connected: {registration.uri}") - - return client, registration - - def get_domains(self, options): - """ - Fetches all domains currently requested - :param options: - :return: - """ - current_app.logger.debug("Fetching domains") - - domains = [] - if "common_name" in options and options["common_name"].strip(): - domains.append(options["common_name"]) - if options.get("extensions"): - for dns_name in options["extensions"]["sub_alt_names"]["names"]: - if dns_name.value not in domains: - domains.append(dns_name.value) - - current_app.logger.debug(f"Got these domains: {domains}") - return domains - - def revoke_certificate(self, certificate, crl_reason=0): - if not self.reuse_account(certificate.authority): - raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") - acme_client, _ = self.setup_acme_client(certificate.authority) - - from cryptography.x509 import load_pem_x509_certificate - fullchain_com = load_pem_x509_certificate(certificate.body.encode("utf-8")) - - try: - acme_client.revoke(fullchain_com, crl_reason) # revocation reason as int (per RFC 5280 section 5.3.1) - except (errors.ConflictError, errors.ClientError, errors.Error) as e: - # Certificate already revoked. - current_app.logger.error("Certificate revocation failed with message: " + e.detail) - metrics.send("acme_revoke_certificate_failure", "counter", 1) - return False - - current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) - metrics.send("acme_revoke_certificate_success", "counter", 1) - return True - - def log_remaining_validation(self, authorizations, acme_account): - for authz in authorizations: - if authz.body.status == STATUS_VALID: - log_data = {'type': authz.body.identifier.typ.name, - 'valid_hours': - int((authz.body.expires - datetime.now(timezone.utc)).total_seconds() / 3600), - 'san': authz.body.identifier.value, - 'account': acme_account} - metrics.send("acme_authz_validation_status", - "gauge", - log_data['valid_hours'], - metric_tags=log_data) - log_data['message'] = "already validated, skipping." - current_app.logger.info(log_data) - - -class AcmeDnsHandler(AcmeHandler): - - def __init__(self): - self.dns_providers_for_domain = {} - try: - self.all_dns_providers = dns_provider_service.get_all_dns_providers() - except Exception as e: - metrics.send("AcmeHandler_init_error", "counter", 1) - capture_exception() - current_app.logger.error(f"Unable to fetch DNS Providers: {e}") - self.all_dns_providers = [] - - def get_all_zones(self, dns_provider): - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - return dns_provider_plugin.get_zones(account_number=account_number) - - def get_dns_challenges(self, host, authorizations): - """Get dns challenges for provided domain - Also indicate if the hostname is already validated - """ - - domain_to_validate, is_wildcard = self.strip_wildcard(host) - dns_challenges = [] - for authz in authorizations: - if not authz.body.identifier.value.lower() == domain_to_validate.lower(): - continue - if is_wildcard and not authz.body.wildcard: - continue - if not is_wildcard and authz.body.wildcard: - continue - # skip valid challenge, as long as this challenge is for the domain_to_validate - if authz.body.status == STATUS_VALID: - metrics.send("get_acme_challenges_already_valid", "counter", 1) - log_data = {"message": "already validated, skipping", "hostname": authz.body.identifier.value} - current_app.logger.info(log_data) - return [], True - - for combo in authz.body.challenges: - if isinstance(combo.chall, challenges.DNS01): - dns_challenges.append(combo) - - return dns_challenges, False - - def get_dns_provider(self, type): - provider_types = { - "cloudflare": cloudflare, - "dyn": dyn, - "route53": route53, - "ultradns": ultradns, - "powerdns": powerdns, - "nsone": nsone - } - provider = provider_types.get(type) - if not provider: - raise UnknownProvider(f"No such DNS provider: {type}") - return provider - - def start_dns_challenge( - self, - acme_client, - account_number, - domain, - target_domain, - dns_provider, - order, - dns_provider_options, - ): - current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.") - - change_ids = [] - cname_delegation = domain != target_domain - # This method will consider and skip valid HTTP01 challenges - dns_challenges, hostname_still_validated = self.get_dns_challenges(domain, order.authorizations) - host_to_validate, _ = self.strip_wildcard(target_domain) - host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) - - if hostname_still_validated: - return - - if not dns_challenges: - capture_exception() - metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) - raise Exception("Unable to determine DNS challenges from authorizations") - - for dns_challenge in dns_challenges: - if not cname_delegation: - host_to_validate = dns_challenge.validation_domain_name(host_to_validate) - - change_id = dns_provider.create_txt_record( - host_to_validate, - dns_challenge.validation(acme_client.net.key), - account_number, - ) - change_ids.append(change_id) - - return AuthorizationRecord( - domain, target_domain, order.authorizations, dns_challenges, change_ids, cname_delegation - ) - - def complete_dns_challenge(self, acme_client, authz_record): - current_app.logger.debug( - "Finalizing DNS challenge for {}".format( - authz_record.authz[0].body.identifier.value - ) - ) - dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) - if not dns_providers: - metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) - raise Exception( - f"No DNS providers found for domain: {authz_record.target_domain}" - ) - - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for change_id in authz_record.change_id: - try: - dns_provider_plugin.wait_for_dns_change( - change_id, account_number=account_number - ) - except Exception: - metrics.send("complete_dns_challenge_error", "counter", 1) - capture_exception() - current_app.logger.debug( - f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " - f"{account_number}", - exc_info=True, - ) - raise - - for dns_challenge in authz_record.dns_challenge: - # abort if the status is already valid, no DNS challenge to complete - if "status" in dns_challenge and dns_challenge["status"] == STATUS_VALID: - metrics.send("acme_challenge_already_valid", "counter", 1) - return - - response = dns_challenge.response(acme_client.net.key) - - verified = response.simple_verify( - dns_challenge.chall, - authz_record.target_domain, - acme_client.net.key.public_key(), - ) - - if not verified: - metrics.send("complete_dns_challenge_verification_error", "counter", 1) - raise ValueError("Failed verification") - - time.sleep(5) - res = acme_client.answer_challenge(dns_challenge, response) - current_app.logger.debug(f"answer_challenge response: {res}") - - def get_authorizations(self, acme_client, order, order_info): - """ The list can be empty if all hostname validations are still valid""" - authorizations = [] - - for domain in order_info.domains: - - # If CNAME exists, set host to the target address - target_domain = domain - if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): - cname_result, _ = self.strip_wildcard(domain) - cname_result = challenges.DNS01().validation_domain_name(cname_result) - cname_result = self.get_cname(cname_result) - if cname_result: - target_domain = cname_result - self.autodetect_dns_providers(target_domain) - metrics.send( - "get_authorizations_cname_delegation_for_domain", "counter", 1, metric_tags={"domain": domain} - ) - - if not self.dns_providers_for_domain.get(target_domain): - metrics.send( - "get_authorizations_no_dns_provider_for_domain", "counter", 1 - ) - raise Exception(f"No DNS providers found for domain: {target_domain}") - - for dns_provider in self.dns_providers_for_domain[target_domain]: - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - authz_record = self.start_dns_challenge( - acme_client, - account_number, - domain, - target_domain, - dns_provider_plugin, - order, - dns_provider.options, - ) - # it can be null, if hostname is still valid - if authz_record: - authorizations.append(authz_record) - return authorizations - - def autodetect_dns_providers(self, domain): - """ - Get DNS providers associated with a domain when it has not been provided for certificate creation. - :param domain: - :return: dns_providers: List of DNS providers that have the correct zone. - """ - self.dns_providers_for_domain[domain] = [] - match_length = 0 - for dns_provider in self.all_dns_providers: - if not dns_provider.domains: - continue - for name in dns_provider.domains: - if name == domain or domain.endswith("." + name): - if len(name) > match_length: - self.dns_providers_for_domain[domain] = [dns_provider] - match_length = len(name) - elif len(name) == match_length: - self.dns_providers_for_domain[domain].append(dns_provider) - - return self.dns_providers_for_domain - - def finalize_authorizations(self, acme_client, authorizations): - for authz_record in authorizations: - self.complete_dns_challenge(acme_client, authz_record) - for authz_record in authorizations: - dns_challenges = authz_record.dns_challenge - for dns_challenge in dns_challenges: - dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_plugin = self.get_dns_provider( - dns_provider.provider_type - ) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) - host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) - if not authz_record.cname_delegation: - host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) - dns_provider_plugin.delete_txt_record( - authz_record.change_id, - account_number, - host_to_validate, - dns_challenge.validation(acme_client.net.key), - ) - - return authorizations - - def cleanup_dns_challenges(self, acme_client, authorizations): - """ - Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called - on an exception - - :param acme_client: - :param authorizations: - :return: - """ - for authz_record in authorizations: - dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_challenges = authz_record.dns_challenge - host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for dns_challenge in dns_challenges: - if not authz_record.cname_delegation: - host_to_validate = dns_challenge.validation_domain_name(host_to_validate) - try: - dns_provider_plugin.delete_txt_record( - authz_record.change_id, - account_number, - host_to_validate, - dns_challenge.validation(acme_client.net.key), - ) - except Exception as e: - # If this fails, it's most likely because the record doesn't exist (It was already cleaned up) - # or we're not authorized to modify it. - metrics.send("cleanup_dns_challenges_error", "counter", 1) - capture_exception() - pass - - def get_cname(self, domain): - """ - :param domain: Domain name to look up a CNAME for. - :return: First CNAME target or False if no CNAME record exists. - """ - try: - result = dns.resolver.query(domain, 'CNAME') - if len(result) > 0: - return str(result[0].target).rstrip('.') - except dns.exception.DNSException: - return False diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py deleted file mode 100644 index ee34f7522c..0000000000 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ /dev/null @@ -1,309 +0,0 @@ -""" -.. module: lemur.plugins.lemur_acme.plugin - :platform: Unix - :synopsis: This module contains the different challenge types for ACME implementations - :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Mathias Petermann -""" -import json -from datetime import datetime, timedelta - -from acme import challenges -from acme.errors import WildcardUnsupportedError -from acme.messages import errors, STATUS_VALID, ERROR_CODES -from botocore.exceptions import ClientError -from flask import current_app -from retrying import retry -from sentry_sdk import capture_exception - -from lemur.authorizations import service as authorization_service -from lemur.common.utils import drop_last_cert_from_chain, csr_to_string -from lemur.constants import ACME_ADDITIONAL_ATTEMPTS -from lemur.destinations import service as destination_service -from lemur.exceptions import LemurException, InvalidConfiguration -from lemur.extensions import metrics -from lemur.plugins.base import plugins -from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler - - -class AcmeChallengeMissmatchError(LemurException): - pass - - -class AcmeChallenge: - """ - This is the base class, all ACME challenges will need to extend, allowing for future extendability - """ - - def create_certificate(self, csr, issuer_options): - """ - Create the new certificate, using the provided CSR and issuer_options. - Right now this is basically a copy of the create_certificate methods in the AcmeHandlers, but should be cleaned - and tried to make use of the deploy and cleanup methods - - :param csr: - :param issuer_options: - :return: - """ - pass - - def deploy(self, challenge, acme_client, validation_target): - """ - In here the challenge validation is fetched and deployed somewhere that it can be validated by the provider - - :param self: - :param challenge: the challenge object, must match for the challenge implementation - :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on - :param validation_target: an identifier for the validation target, e.g. the name of a DNS provider - """ - raise NotImplementedError - - def cleanup(self, challenge, acme_client, validation_target): - """ - Ideally the challenge should be cleaned up, after the validation is done - :param challenge: Needed to identify the challenge to be removed - :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on - :param validation_target: Needed to remove the validation - """ - raise NotImplementedError - - -class AcmeHttpChallenge(AcmeChallenge): - challengeType = challenges.HTTP01 - - def create_certificate(self, csr, issuer_options): - """ - Creates an ACME certificate using the HTTP-01 challenge. - - :param csr: - :param issuer_options: - :return: :raise Exception: - """ - self.acme = AcmeHandler() - authority = issuer_options.get("authority") - acme_client, registration = self.acme.setup_acme_client(authority) - - orderr = acme_client.new_order(csr_to_string(csr)) - - chall = [] - deployed_challenges = [] - all_pre_validated = True - for authz in orderr.authorizations: - # Choosing challenge. - if authz.body.status != STATUS_VALID: - all_pre_validated = False - # authz.body.challenges is a set of ChallengeBody objects. - for i in authz.body.challenges: - # Find the supported challenge. - if isinstance(i.chall, challenges.HTTP01): - chall.append(i) - else: - metrics.send("get_acme_challenges_already_valid", "counter", 1) - log_data = {"message": "already validated, skipping", "hostname": authz.body.identifier.value} - current_app.logger.info(log_data) - - if len(chall) == 0 and not all_pre_validated: - raise Exception(f'HTTP-01 challenge was not offered by the CA server at {orderr.uri}') - elif not all_pre_validated: - validation_target = None - for option in json.loads(issuer_options["authority"].options): - if option["name"] == "tokenDestination": - validation_target = option["value"] - - if validation_target is None: - raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') - - for challenge in chall: - try: - response = self.deploy(challenge, acme_client, validation_target) - deployed_challenges.append(challenge.chall.path) - acme_client.answer_challenge(challenge, response) - except Exception as e: - current_app.logger.error(e) - raise Exception('Failure while trying to deploy token to configure destination. See logs for more information') - - current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") - - try: - deadline = datetime.now() + timedelta(seconds=90) - orderr = acme_client.poll_authorizations(orderr, deadline) - finalized_orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) - - except errors.ValidationError as validationError: - error_message = "Validation error occurred, can\'t complete challenges. See logs for more information." - for authz in validationError.failed_authzrs: - for chall in authz.body.challenges: - if chall.error: - error_message = f"ValidationError occurred of type: {chall.error.typ}, " \ - f"with message: {ERROR_CODES[chall.error.code]}, " \ - f"detail: {chall.error.detail}" - current_app.logger.error(error_message) - - raise Exception(error_message) - - pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem, - finalized_orderr.alternative_fullchains_pem) - - if "drop_last_cert_from_chain" in authority.options: - for option in json.loads(authority.options): - if option["name"] == "drop_last_cert_from_chain" and option["value"] is True: - # skipping the last element - pem_certificate_chain = drop_last_cert_from_chain(pem_certificate_chain) - - self.acme.log_remaining_validation(finalized_orderr.authorizations, - acme_client.net.account.uri.replace('https://', '')) - - if len(deployed_challenges) != 0: - for token_path in deployed_challenges: - self.cleanup(token_path, validation_target) - - # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate - return pem_certificate, pem_certificate_chain, None - - def deploy(self, challenge, acme_client, validation_target): - - if not isinstance(challenge.chall, challenges.HTTP01): - raise AcmeChallengeMissmatchError( - 'The provided challenge is not of type HTTP01, but instead of type {}'.format( - challenge.__class__.__name__)) - - destination = destination_service.get(validation_target) - - if destination is None: - raise Exception( - f'Couldn\'t find the destination with name {validation_target}. Cant complete HTTP01 challenge') - - destination_plugin = plugins.get(destination.plugin_name) - - response, validation = challenge.response_and_validation(acme_client.net.key) - - destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options) - current_app.logger.info("Uploaded HTTP-01 challenge token.") - - return response - - def cleanup(self, token_path, validation_target): - destination = destination_service.get(validation_target) - - if destination is None: - current_app.logger.info( - f'Couldn\'t find the destination with name {validation_target}, won\'t cleanup the challenge') - - destination_plugin = plugins.get(destination.plugin_name) - - destination_plugin.delete_acme_token(token_path, destination.options) - current_app.logger.info("Cleaned up HTTP-01 challenge token.") - - -class AcmeDnsChallenge(AcmeChallenge): - challengeType = challenges.DNS01 - - def create_certificate(self, csr, issuer_options): - """ - Creates an ACME certificate. - - :param csr: - :param issuer_options: - :return: :raise Exception: - """ - self.acme = AcmeDnsHandler() - authority = issuer_options.get("authority") - create_immediately = issuer_options.get("create_immediately", False) - acme_client, registration = self.acme.setup_acme_client(authority) - domains = self.acme.get_domains(issuer_options) - dns_provider = issuer_options.get("dns_provider", {}) - - if dns_provider: - for domain in domains: - # Currently, we only support specifying one DNS provider per certificate, even if that - # certificate has multiple SANs that may belong to different provid - self.acme.dns_providers_for_domain[domain] = [dns_provider] - - credentials = json.loads(dns_provider.credentials) - current_app.logger.debug( - f"Using DNS provider: {dns_provider.provider_type}" - ) - account_number = credentials.get("account_id") - provider_type = dns_provider.provider_type - if provider_type == "route53" and not account_number: - error = "Route53 DNS Provider {} does not have an account number configured.".format( - dns_provider.name - ) - current_app.logger.error(error) - raise InvalidConfiguration(error) - else: - dns_provider = {} - account_number = None - provider_type = None - - for domain in domains: - self.acme.autodetect_dns_providers(domain) - - # Create pending authorizations that we'll need to do the creation - dns_authorization = authorization_service.create( - account_number, domains, provider_type - ) - - if not create_immediately: - # Return id of the DNS Authorization - return None, None, dns_authorization.id - - pem_certificate, pem_certificate_chain = self.create_certificate_immediately( - acme_client, dns_authorization, csr - ) - - if "drop_last_cert_from_chain" in authority.options \ - and authority.options.get("drop_last_cert_from_chain") is True: - # skipping the last element - pem_certificate_chain = drop_last_cert_from_chain(pem_certificate_chain) - - # TODO add external ID (if possible) - return pem_certificate, pem_certificate_chain, None - - @retry(stop_max_attempt_number=ACME_ADDITIONAL_ATTEMPTS, wait_fixed=5000) - def create_certificate_immediately(self, acme_client, order_info, csr): - try: - order = acme_client.new_order(csr_to_string(csr)) - except WildcardUnsupportedError: - metrics.send("create_certificte_immediately_wildcard_unsupported", "counter", 1) - raise Exception( - "The currently selected ACME CA endpoint does" - " not support issuing wildcard certificates." - ) - - try: - authorizations = self.acme.get_authorizations( - acme_client, order, order_info - ) - except ClientError: - capture_exception() - metrics.send("create_certificate_immediately_error", "counter", 1) - - current_app.logger.error( - f"Unable to resolve cert for domains: {', '.join(order_info.domains)}", exc_info=True - ) - return False - - authorizations = self.acme.finalize_authorizations(acme_client, authorizations) - - return self.acme.request_certificate( - acme_client, authorizations, order - ) - - def deploy(self, challenge, acme_client, validation_target): - pass - - def cleanup(self, authorizations, acme_client, validation_target=None): - """ - Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called - on an exception - - :param authorizations: all the authorizations to be cleaned up - :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on - :param validation_target: Unused right now - :return: - """ - acme = AcmeDnsHandler() - acme.cleanup_dns_challenges(acme_client, authorizations) diff --git a/lemur/plugins/lemur_acme/cloudflare.py b/lemur/plugins/lemur_acme/cloudflare.py deleted file mode 100644 index af13867e1e..0000000000 --- a/lemur/plugins/lemur_acme/cloudflare.py +++ /dev/null @@ -1,81 +0,0 @@ -import time - -import CloudFlare -from flask import current_app - - -def cf_api_call(): - cf_key = current_app.config.get("ACME_CLOUDFLARE_KEY", "") - cf_email = current_app.config.get("ACME_CLOUDFLARE_EMAIL", "") - return CloudFlare.CloudFlare(email=cf_email, token=cf_key) - - -def find_zone_id(host): - elements = host.split(".") - cf = cf_api_call() - - n = 1 - - while n < 5: - n = n + 1 - domain = ".".join(elements[-n:]) - current_app.logger.debug(f"Trying to get ID for zone {domain}") - - try: - zone = cf.zones.get(params={"name": domain, "per_page": 1}) - except Exception as e: - current_app.logger.error("Cloudflare API error: %s" % e) - pass - - if len(zone) == 1: - break - - if len(zone) == 0: - current_app.logger.error("No zone found") - return - else: - return zone[0]["id"] - - -def wait_for_dns_change(change_id, account_number=None): - cf = cf_api_call() - zone_id, record_id = change_id - while True: - r = cf.zones.get(zone_id, record_id) - current_app.logger.debug("Record status: %s" % r["status"]) - if r["status"] == "active": - break - time.sleep(1) - return - - -def create_txt_record(host, value, account_number): - cf = cf_api_call() - zone_id = find_zone_id(host) - if not zone_id: - return - - txt_record = {"name": host, "type": "TXT", "content": value} - - current_app.logger.debug( - f"Creating TXT record {host} with value {value}" - ) - - try: - r = cf.zones.dns_records.post(zone_id, data=txt_record) - except Exception as e: - current_app.logger.error( - "/zones.dns_records.post {}: {}".format(txt_record["name"], e) - ) - return zone_id, r["id"] - - -def delete_txt_record(change_ids, account_number, host, value): - cf = cf_api_call() - for change_id in change_ids: - zone_id, record_id = change_id - current_app.logger.debug(f"Removing record with id {record_id}") - try: - cf.zones.dns_records.delete(zone_id, record_id) - except Exception as e: - current_app.logger.error("/zones.dns_records.post: %s" % e) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py deleted file mode 100644 index 5cf40a95d4..0000000000 --- a/lemur/plugins/lemur_acme/dyn.py +++ /dev/null @@ -1,286 +0,0 @@ -import time - -import dns -import dns.exception -import dns.name -import dns.query -import dns.resolver -from dyn.tm.errors import ( - DynectCreateError, - DynectDeleteError, - DynectGetError, - DynectUpdateError, -) -from dyn.tm.session import DynectSession -from dyn.tm.zones import Node, Zone, get_all_zones -from flask import current_app -from sentry_sdk import capture_exception - -from lemur.extensions import metrics - - -def get_dynect_session(): - try: - dynect_session = DynectSession( - current_app.config.get("ACME_DYN_CUSTOMER_NAME", ""), - current_app.config.get("ACME_DYN_USERNAME", ""), - current_app.config.get("ACME_DYN_PASSWORD", ""), - ) - except Exception as e: - capture_exception() - metrics.send("get_dynect_session_fail", "counter", 1) - current_app.logger.debug("Unable to establish connection to Dyn", exc_info=True) - raise - return dynect_session - - -def _has_dns_propagated(fqdn, token): - txt_records = [] - try: - dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [get_authoritative_nameserver(fqdn)] - dns_response = dns_resolver.query(fqdn, "TXT") - for rdata in dns_response: - for txt_record in rdata.strings: - txt_records.append(txt_record.decode("utf-8")) - except dns.exception.DNSException: - metrics.send("has_dns_propagated_fail", "counter", 1, metric_tags={"dns": fqdn}) - return False - - for txt_record in txt_records: - if txt_record == token: - metrics.send("has_dns_propagated_success", "counter", 1, metric_tags={"dns": fqdn}) - return True - - return False - - -def wait_for_dns_change(change_id, account_number=None): - fqdn, token = change_id - number_of_attempts = 20 - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token) - current_app.logger.debug(f"Record status for fqdn: {fqdn}: {status}") - if status: - metrics.send("wait_for_dns_change_success", "counter", 1, metric_tags={"dns": fqdn}) - break - time.sleep(10) - if not status: - # TODO: Delete associated DNS text record here - metrics.send("wait_for_dns_change_fail", "counter", 1, metric_tags={"dns": fqdn}) - capture_exception(extra={"fqdn": str(fqdn), "txt_record": str(token)}) - metrics.send( - "wait_for_dns_change_error", - "counter", - 1, - metric_tags={"fqdn": fqdn, "txt_record": token}, - ) - return - - -def get_zone_name(domain): - zones = get_all_zones() - - zone_name = "" - - for z in zones: - if domain.endswith(z.name): - # Find the most specific zone possible for the domain - # Ex: If fqdn is a.b.c.com, there is a zone for c.com, - # and a zone for b.c.com, we want to use b.c.com. - if z.name.count(".") > zone_name.count("."): - zone_name = z.name - if not zone_name: - metrics.send("dyn_no_zone_name", "counter", 1) - raise Exception(f"No Dyn zone found for domain: {domain}") - return zone_name - - -def get_zones(account_number): - get_dynect_session() - zones = get_all_zones() - zone_list = [] - for zone in zones: - zone_list.append(zone.name) - return zone_list - - -def create_txt_record(domain, token, account_number): - get_dynect_session() - zone_name = get_zone_name(domain) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = f"{node_name}.{zone_name}" - zone = Zone(zone_name) - - try: - zone.add_record( - node_name, record_type="TXT", txtdata=f'"{token}"', ttl=5 - ) - zone.publish() - current_app.logger.debug( - f"TXT record created: {fqdn}, token: {token}" - ) - except (DynectCreateError, DynectUpdateError) as e: - if "Cannot duplicate existing record data" in e.message: - current_app.logger.debug( - "Unable to add record. Domain: {}. Token: {}. " - "Record already exists: {}".format(domain, token, e), - exc_info=True, - ) - else: - metrics.send("create_txt_record_error", "counter", 1) - capture_exception() - raise - - change_id = (fqdn, token) - return change_id - - -def delete_txt_record(change_id, account_number, domain, token): - get_dynect_session() - if not domain: - current_app.logger.debug("delete_txt_record: No domain passed") - return - - zone_name = get_zone_name(domain) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = f"{node_name}.{zone_name}" - - zone = Zone(zone_name) - node = Node(zone_name, fqdn) - - try: - all_txt_records = node.get_all_records_by_type("TXT") - except DynectGetError: - metrics.send("delete_txt_record_geterror", "counter", 1) - # No Text Records remain or host is not in the zone anymore because all records have been deleted. - return - for txt_record in all_txt_records: - if txt_record.txtdata == (f"{token}"): - current_app.logger.debug(f"Deleting TXT record name: {fqdn}") - try: - txt_record.delete() - except DynectDeleteError: - capture_exception( - extra={ - "fqdn": str(fqdn), - "zone_name": str(zone_name), - "node_name": str(node_name), - "txt_record": str(txt_record.txtdata), - } - ) - metrics.send( - "delete_txt_record_deleteerror", - "counter", - 1, - metric_tags={"fqdn": fqdn, "txt_record": txt_record.txtdata}, - ) - - try: - zone.publish() - except DynectUpdateError: - capture_exception( - extra={ - "fqdn": str(fqdn), - "zone_name": str(zone_name), - "node_name": str(node_name), - "txt_record": str(txt_record.txtdata), - } - ) - metrics.send( - "delete_txt_record_publish_error", - "counter", - 1, - metric_tags={"fqdn": str(fqdn), "txt_record": str(txt_record.txtdata)}, - ) - - -def delete_acme_txt_records(domain): - get_dynect_session() - if not domain: - current_app.logger.debug("delete_acme_txt_records: No domain passed") - return - acme_challenge_string = "_acme-challenge" - if not domain.startswith(acme_challenge_string): - current_app.logger.debug( - "delete_acme_txt_records: Domain {} doesn't start with string {}. " - "Cowardly refusing to delete TXT records".format( - domain, acme_challenge_string - ) - ) - return - - zone_name = get_zone_name(domain) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = f"{node_name}.{zone_name}" - - zone = Zone(zone_name) - node = Node(zone_name, fqdn) - - all_txt_records = node.get_all_records_by_type("TXT") - for txt_record in all_txt_records: - current_app.logger.debug(f"Deleting TXT record name: {fqdn}") - try: - txt_record.delete() - except DynectDeleteError: - capture_exception( - extra={ - "fqdn": str(fqdn), - "zone_name": str(zone_name), - "node_name": str(node_name), - "txt_record": str(txt_record.txtdata), - } - ) - metrics.send( - "delete_txt_record_deleteerror", - "counter", - 1, - metric_tags={"fqdn": fqdn, "txt_record": txt_record.txtdata}, - ) - zone.publish() - - -def get_authoritative_nameserver(domain): - if current_app.config.get("ACME_DYN_GET_AUTHORATATIVE_NAMESERVER"): - n = dns.name.from_text(domain) - - depth = 2 - default = dns.resolver.get_default_resolver() - nameserver = default.nameservers[0] - - last = False - while not last: - s = n.split(depth) - - last = s[0].to_unicode() == "@" - sub = s[1] - - query = dns.message.make_query(sub, dns.rdatatype.NS) - response = dns.query.udp(query, nameserver) - - rcode = response.rcode() - if rcode != dns.rcode.NOERROR: - metrics.send("get_authoritative_nameserver_error", "counter", 1) - if rcode == dns.rcode.NXDOMAIN: - raise Exception("%s does not exist." % sub) - else: - raise Exception("Error %s" % dns.rcode.to_text(rcode)) - - if len(response.authority) > 0: - rrset = response.authority[0] - else: - rrset = response.answer[0] - - rr = rrset[0] - if rr.rdtype != dns.rdatatype.SOA: - authority = rr.target - nameserver = default.query(authority).rrset[0].to_text() - - depth += 1 - - return nameserver - else: - return "8.8.8.8" diff --git a/lemur/plugins/lemur_acme/nsone.py b/lemur/plugins/lemur_acme/nsone.py deleted file mode 100644 index b66c802b2e..0000000000 --- a/lemur/plugins/lemur_acme/nsone.py +++ /dev/null @@ -1,360 +0,0 @@ -"""ACME DNS providor for NS1""" -import json -import inspect -import time -import requests - -from flask import current_app -from sentry_sdk import capture_exception -from lemur.common import utils -import lemur.dns_providers.util as dnsutil -from lemur.extensions import metrics - -REQUIRED_VARIABLES = [ - "ACME_NSONE_KEY", -] - - -def get_zones(): - """ - Retrieve authoritative zones from the NS1 API and return a list of zones - - :raise: Exception - :return: list of Zone Objects - """ - _check_conf() - path = f'{"/v1/zones"}' - zones = [] - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function - } - try: - records = _get(path) - log_data["message"] = "Retrieved Zones Successfully" - current_app.logger.debug(log_data) - except Exception as err: - capture_exception() - log_data["message"] = "Failed to Retrieve Zone Data" - log_data["error"] = err - current_app.logger.debug(log_data) - raise - for record in records: - zone = record['zone'] - if record['primary']['enabled']: - zones.append(zone) - return zones - - -def create_txt_record(domain, token, account_number=None): - """ - Create a TXT record for the given domain and token and return a change_id tuple - - :param domain: FQDN - :param token: challenge value - :param account_number: - :return: tuple of domain/token - """ - _check_conf() - - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - "account": account_number, - } - # Create new record - answer = {"answer": [token]} - # Get current records - found = False - records = _get_txt_records(domain) - for each in records['answers']: - if token in each['answer']: - found = True - break - if not found: - records['answers'].append(answer) - log_data["records"] = records - try: - if 'id' in records: - _patch_txt_records(domain, records) - else: - _patch_txt_records(domain, records, patch=False) - log_data["message"] = "TXT record(s) successfully created" - current_app.logger.debug(log_data) - except Exception as err: - capture_exception() - log_data["Exception"] = err - log_data["message"] = "Unable to create TXT record(s)" - current_app.logger.debug(log_data) - change_id = (domain, token) - return change_id - - -def wait_for_dns_change(change_id, account_number=None): - """ - Checks the authoritative DNS Server to see if changes have propagated. - - :param change_id: tuple of domain/token - :param account_number: - :return: - """ - _check_conf() - domain, token = change_id - number_of_attempts = 3 - zone = _get_zone_name(domain) - nameserver = dnsutil.get_authoritative_nameserver(zone) - record_found = False - attempts = 0 - while attempts < number_of_attempts: - txt_records = dnsutil.get_dns_records(domain, "TXT", nameserver) - for txt_record in txt_records: - if txt_record == token: - record_found = True - break - if record_found: - break - time.sleep(5) - attempts += 1 - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "status": record_found, - "account": account_number, - "message": "Record status on NS1 authoritative server", - } - current_app.logger.debug(log_data) - if record_found: - metrics.send( - f"{function}.success", - "counter", 1, - metric_tags={"fqdn": domain, "txt_record": token} - ) - else: - metrics.send( - f"{function}.fail", - "counter", 1, - metric_tags={"fqdn": domain, "txt_record": token} - ) - - -def delete_txt_record(change_id, account_number, domain, token): - """ - Delete the TXT record for the given domain and token - - :param change_id: tuple of domain/token - :param account_number: - :param domain: FQDN - :param token: challenge to delete - :return: - """ - _check_conf() - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - "change": change_id, - "account": account_number, - } - zone = _get_zone_name(domain) - records = _get_txt_records(domain) - found = False - for each in records['answers']: - if token in each['answer']: - found = True - each['answer'].remove(token) - if len(each['answer']) == 0: - try: - path = f"/v1/zones/{zone}/{domain}/TXT" - _delete(path) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as err: - capture_exception() - log_data["Exception"] = err - log_data["message"] = "Unable to delete TXT record" - current_app.logger.debug(log_data) - else: - try: - _patch_txt_records(domain, records) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as err: - capture_exception() - log_data["Exception"] = err - log_data["message"] = "Unable to delete TXT record" - current_app.logger.debug(log_data) - # Since the matching token is not in DNS, there is nothing to delete - if not found: - log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records" - current_app.logger.debug(log_data) - return - - -def _check_conf(): - """ - Verifies required configuration variables are set - - :return: - """ - utils.validate_conf(current_app, REQUIRED_VARIABLES) - - -def _generate_header(): - """ - Generate a NS1 API header and return it as a dictionary - - :return: Dict of header parameters - """ - api_key = current_app.config.get("ACME_NSONE_KEY") - headers = {'X-NSONE-Key': api_key} - return headers - - -def _get_zone_name(domain): - """ - Get most specific matching zone for the given domain and return as a String - - :param domain: FQDN - :return: FQDN of domain - """ - zones = get_zones() - zone_name = "" - for zone in zones: - if domain.endswith("." + zone) or domain == zone: - if zone.count(".") > zone_name.count("."): - zone_name = zone - if not zone_name: - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "message": "No NS1 zone name found.", - } - current_app.logger.debug(log_data) - metrics.send(f"{function}.fail", "counter", 1) - return zone_name - - -def _get_txt_records(domain): - """ - Retrieve TXT records for a given domain and return list of Record Objects - - :param domain: FQDN - :return: list of Record objects - """ - zone = _get_zone_name(domain) - path = f"/v1/zones/{zone}/{domain}/TXT" - - function = inspect.currentframe().f_code.co_name - log_data = { - "function": function - } - try: - records = _get(path) - log_data["message"] = "Retrieved TXT Records Successfully" - log_data['records'] = records - current_app.logger.debug(log_data) - - except Exception as err: - capture_exception() - log_data["Exception"] = err - log_data["message"] = "Failed to Retrieve TXT Records" - current_app.logger.debug(log_data) - records = {} - records['domain'] = domain - records['zone'] = zone - records['type'] = 'TXT' - records['answers'] = [] - - return records - - -def _get(path, params=None): - """ - Execute a GET request on the given URL (base_uri + path) and return response as JSON object - - :param path: Relative URL path - :param params: additional parameters - :return: json response - """ - base_uri = 'https://api.nsone.net' - resp = requests.get( - f"{base_uri}{path}", - headers=_generate_header(), - params=params, - verify=True - ) - resp.raise_for_status() - return resp.json() - - -def _patch_txt_records(domain, records, patch=True): - """ - Send Patch or Put request to NS1 Server - - :param domain: FQDN - :param account_number: - :param records: List of Record objects - :return: - """ - # Create Txt Records - zone = _get_zone_name(domain) - path = f"/v1/zones/{zone}/{domain}/TXT" - if patch: - _patch(path, records) - else: - _put(path, records) - - -def _patch(path, payload): - """ - Execute a Patch to update the given URL (base_uri + path) with given payload - - :param path: - :param payload: - :return: - """ - base_uri = 'https://api.nsone.net' - resp = requests.post( - f"{base_uri}{path}", - data=json.dumps(payload), - headers=_generate_header(), - verify=True - ) - resp.raise_for_status() - - -def _put(path, payload): - """ - Execute a Put to add the given URL (base_uri + path) with payload - - :param path: - :param payload: - :return: - """ - base_uri = 'https://api.nsone.net' - resp = requests.put( - f"{base_uri}{path}", - data=json.dumps(payload), - headers=_generate_header(), - verify=True - ) - resp.raise_for_status() - - -def _delete(path): - """ - Execute a Delete requests on the given URL (base_uri + path) - - """ - base_uri = 'https://api.nsone.net' - resp = requests.delete( - f"{base_uri}{path}", - headers=_generate_header(), - verify=True - ) - resp.raise_for_status() diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py deleted file mode 100644 index a57faae608..0000000000 --- a/lemur/plugins/lemur_acme/plugin.py +++ /dev/null @@ -1,491 +0,0 @@ -""" -.. module: lemur.plugins.lemur_acme.plugin - :platform: Unix - :synopsis: This module is responsible for communicating with an ACME CA. - :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - - Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py - -.. moduleauthor:: Kevin Glisson -.. moduleauthor:: Mikhail Khodorovskiy -.. moduleauthor:: Curtis Castrapel -""" - -from urllib.parse import urlparse - -from acme.errors import PollError, WildcardUnsupportedError -from acme.messages import Error as AcmeError -from botocore.exceptions import ClientError -from flask import current_app -from sentry_sdk import capture_exception - -from lemur.authorizations import service as authorization_service -from lemur.common.utils import check_validation, drop_last_cert_from_chain, csr_to_string -from lemur.constants import CRLReason, EMAIL_RE -from lemur.dns_providers import service as dns_provider_service -from lemur.exceptions import InvalidConfiguration -from lemur.extensions import metrics -from lemur.plugins import lemur_acme as acme -from lemur.plugins.bases import IssuerPlugin -from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler -from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge - - -def validate_acme_url(url): - """Reject acme_url values that are not in the configured allowlist.""" - allowed_hosts = current_app.config.get( - "ACME_DIRECTORY_HOST_ALLOWLIST", - { - "acme-v02.api.letsencrypt.org", - "acme-staging-v02.api.letsencrypt.org", - "dv.acme-v02.api.pki.goog", - }, - ) - parsed = urlparse(url) - if parsed.scheme != "https" or parsed.hostname not in allowed_hosts: - raise InvalidConfiguration( - f"acme_url host not in ACME_DIRECTORY_HOST_ALLOWLIST: {parsed.hostname}" - ) - - -class ACMEIssuerPlugin(IssuerPlugin): - title = "Acme" - slug = "acme-issuer" - description = ( - "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge" - ) - version = acme.VERSION - - author = "Netflix" - author_url = "https://github.com/netflix/lemur.git" - - options = [ - { - "name": "acme_url", - "type": "str", - "required": True, - "validation": check_validation(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$_@.&+-]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"), - "helpMessage": "ACME resource URI. Must be a valid web url starting with http[s]://", - }, - { - "name": "telephone", - "type": "str", - "default": "", - "helpMessage": "Telephone to use", - }, - { - "name": "email", - "type": "str", - "default": "", - "validation": EMAIL_RE.pattern, - "helpMessage": "Email to use", - }, - { - "name": "certificate", - "type": "textarea", - "default": "", - "validation": check_validation("^-----BEGIN CERTIFICATE-----"), - "helpMessage": "ACME root certificate", - }, - { - "name": "store_account", - "type": "bool", - "required": False, - "helpMessage": "Disable to create a new account for each ACME request", - "default": False, - }, - { - "name": "eab_kid", - "type": "str", - "required": False, - "helpMessage": "Key identifier for the external account.", - }, - { - "name": "eab_hmac_key", - "type": "str", - "required": False, - "helpMessage": "HMAC key for the external account.", - }, - { - "name": "acme_private_key", - "type": "textarea", - "default": "", - "required": False, - "helpMessage": "Account Private Key. Will be encrypted.", - }, - { - "name": "acme_regr", - "type": "textarea", - "default": "", - "required": False, - "helpMessage": "Account Registration", - }, - { - "name": "drop_last_cert_from_chain", - "type": "bool", - "required": False, - "helpMessage": "Drops the last certificate, i.e., the Cross Signed root, from the Chain", - "default": False, - } - ] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def get_ordered_certificate(self, pending_cert): - self.acme = AcmeDnsHandler() - acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) - order_info = authorization_service.get(pending_cert.external_id) - if pending_cert.dns_provider_id: - dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) - - for domain in order_info.domains: - # Currently, we only support specifying one DNS provider per certificate, even if that - # certificate has multiple SANs that may belong to different providers. - self.acme.dns_providers_for_domain[domain] = [dns_provider] - else: - for domain in order_info.domains: - self.acme.autodetect_dns_providers(domain) - - try: - order = acme_client.new_order(csr_to_string(pending_cert.csr)) - except WildcardUnsupportedError: - metrics.send("get_ordered_certificate_wildcard_unsupported", "counter", 1) - raise Exception( - "The currently selected ACME CA endpoint does" - " not support issuing wildcard certificates." - ) - try: - authorizations = self.acme.get_authorizations( - acme_client, order, order_info - ) - except ClientError: - capture_exception() - metrics.send("get_ordered_certificate_error", "counter", 1) - current_app.logger.error( - f"Unable to resolve pending cert: {pending_cert.name}", exc_info=True - ) - return False - - authorizations = self.acme.finalize_authorizations(acme_client, authorizations) - pem_certificate, pem_certificate_chain = self.acme.request_certificate( - acme_client, authorizations, order - ) - cert = { - "body": "\n".join(str(pem_certificate).splitlines()), - "chain": "\n".join(str(pem_certificate_chain).splitlines()), - "external_id": str(pending_cert.external_id), - } - - if self.options and "drop_last_cert_from_chain" in self.options \ - and self.options.get("drop_last_cert_from_chain") is True: - # skipping the last element - cert["chain"] = drop_last_cert_from_chain(cert["chain"]) - - return cert - - def get_ordered_certificates(self, pending_certs): - self.acme = AcmeDnsHandler() - self.acme_dns_challenge = AcmeDnsChallenge() - pending = [] - certs = [] - for pending_cert in pending_certs: - try: - acme_client, registration = self.acme.setup_acme_client( - pending_cert.authority - ) - order_info = authorization_service.get(pending_cert.external_id) - if pending_cert.dns_provider_id: - dns_provider = dns_provider_service.get( - pending_cert.dns_provider_id - ) - - for domain in order_info.domains: - # Currently, we only support specifying one DNS provider per certificate, even if that - # certificate has multiple SANs that may belong to different providers. - self.acme.dns_providers_for_domain[domain] = [dns_provider] - else: - for domain in order_info.domains: - self.acme.autodetect_dns_providers(domain) - - try: - order = acme_client.new_order(csr_to_string(pending_cert.csr)) - except WildcardUnsupportedError: - capture_exception() - metrics.send( - "get_ordered_certificates_wildcard_unsupported_error", - "counter", - 1, - ) - raise Exception( - "The currently selected ACME CA endpoint does" - " not support issuing wildcard certificates." - ) - - authorizations = self.acme.get_authorizations( - acme_client, order, order_info - ) - - pending.append( - { - "acme_client": acme_client, - "authorizations": authorizations, - "pending_cert": pending_cert, - "order": order, - } - ) - except (ClientError, ValueError, Exception) as e: - capture_exception() - metrics.send( - "get_ordered_certificates_pending_creation_error", "counter", 1 - ) - current_app.logger.error( - f"Unable to resolve pending cert: {pending_cert}", exc_info=True - ) - - error = e - if globals().get("order") and order: - error += f" Order uri: {order.uri}" - certs.append( - {"cert": False, "pending_cert": pending_cert, "last_error": e} - ) - - for entry in pending: - try: - entry["authorizations"] = self.acme.finalize_authorizations( - entry["acme_client"], entry["authorizations"] - ) - pem_certificate, pem_certificate_chain = self.acme.request_certificate( - entry["acme_client"], entry["authorizations"], entry["order"] - ) - - cert = { - "body": "\n".join(str(pem_certificate).splitlines()), - "chain": "\n".join(str(pem_certificate_chain).splitlines()), - "external_id": str(entry["pending_cert"].external_id), - } - certs.append({"cert": cert, "pending_cert": entry["pending_cert"]}) - - if self.options and "drop_last_cert_from_chain" in self.options \ - and self.options.get("drop_last_cert_from_chain") is True: - cert["chain"] = drop_last_cert_from_chain(cert["chain"]) - - except (PollError, AcmeError, Exception) as e: - capture_exception() - metrics.send("get_ordered_certificates_resolution_error", "counter", 1) - order_url = order.uri - error = f"{e}. Order URI: {order_url}" - current_app.logger.error( - f"Unable to resolve pending cert: {pending_cert}. " - f"Check out {order_url} for more information.", - exc_info=True, - ) - certs.append( - { - "cert": False, - "pending_cert": entry["pending_cert"], - "last_error": error, - } - ) - # Ensure DNS records get deleted - self.acme_dns_challenge.cleanup( - entry["authorizations"], entry["acme_client"] - ) - return certs - - def create_certificate(self, csr, issuer_options): - """ - Creates an ACME certificate using the DNS-01 challenge. - - :param csr: - :param issuer_options: - :return: :raise Exception: - """ - acme_dns_challenge = AcmeDnsChallenge() - - return acme_dns_challenge.create_certificate(csr, issuer_options) - - @staticmethod - def create_authority(options): - """ - Creates an authority, this authority is then used by Lemur to allow a user - to specify which Certificate Authority they want to sign their certificate. - - :param options: - :return: - """ - name = "acme_" + "_".join(options['name'].split(" ")) + "_admin" - role = {"username": "", "password": "", "name": name} - - plugin_options = options.get("plugin", {}).get("plugin_options") - if not plugin_options: - error = f"Invalid options for lemur_acme plugin: {options}" - current_app.logger.error(error) - raise InvalidConfiguration(error) - # Define static acme_root based off configuration variable by default. However, if user has passed a - # certificate, use this certificate as the root. - acme_root = current_app.config.get("ACME_ROOT") - for option in plugin_options: - if option.get("name") == "certificate": - acme_root = option.get("value") - if option.get("name") == "acme_url": - validate_acme_url(option.get("value", "")) - return acme_root, "", [role] - - def cancel_ordered_certificate(self, pending_cert, **kwargs): - # Needed to override issuer function. - pass - - def revoke_certificate(self, certificate, reason): - self.acme = AcmeDnsHandler() - crl_reason = CRLReason.unspecified - if "crl_reason" in reason: - crl_reason = CRLReason[reason["crl_reason"]] - - return self.acme.revoke_certificate(certificate, crl_reason.value) - - -class ACMEHttpIssuerPlugin(IssuerPlugin): - title = "Acme HTTP-01" - slug = "acme-http-issuer" - description = ( - "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge" - ) - version = acme.VERSION - - author = "Netflix" - author_url = "https://github.com/netflix/lemur.git" - - options = [ - { - "name": "acme_url", - "type": "str", - "required": True, - "validation": check_validation(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$_@.&+-]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"), - "helpMessage": "Must be a valid web url starting with http[s]://", - }, - { - "name": "telephone", - "type": "str", - "default": "", - "helpMessage": "Telephone to use", - }, - { - "name": "email", - "type": "str", - "default": "", - "validation": EMAIL_RE.pattern, - "helpMessage": "Email to use", - }, - { - "name": "certificate", - "type": "textarea", - "default": "", - "validation": check_validation("^-----BEGIN CERTIFICATE-----"), - "helpMessage": "ACME root Certificate", - }, - { - "name": "store_account", - "type": "bool", - "required": False, - "helpMessage": "Disable to create a new account for each ACME request", - "default": False, - }, - { - "name": "eab_kid", - "type": "str", - "default": "", - "required": False, - "helpMessage": "Key identifier for the external account.", - }, - { - "name": "eab_hmac_key", - "type": "str", - "default": "", - "required": False, - "helpMessage": "HMAC key for the external account.", - }, - { - "name": "acme_private_key", - "type": "textarea", - "default": "", - "required": False, - "helpMessage": "Account Private Key. Will be encrypted.", - }, - { - "name": "acme_regr", - "type": "textarea", - "default": "", - "required": False, - "helpMessage": "Account Registration", - }, - { - "name": "tokenDestination", - "type": "destinationSelect", - "required": True, - "helpMessage": "The destination to use to deploy the token.", - }, - { - "name": "drop_last_cert_from_chain", - "type": "bool", - "required": False, - "helpMessage": "Drops the last certificate, i.e., the Cross Signed root, from the Chain", - "default": False, - } - ] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def create_certificate(self, csr, issuer_options): - """ - Creates an ACME certificate using the HTTP-01 challenge. - - :param csr: - :param issuer_options: - :return: :raise Exception: - """ - acme_http_challenge = AcmeHttpChallenge() - - return acme_http_challenge.create_certificate(csr, issuer_options) - - @staticmethod - def create_authority(options): - """ - Creates an authority, this authority is then used by Lemur to allow a user - to specify which Certificate Authority they want to sign their certificate. - - :param options: - :return: - """ - name = "acme_" + "_".join(options['name'].split(" ")) + "_admin" - role = {"username": "", "password": "", "name": name} - - plugin_options = options.get("plugin", {}).get("plugin_options") - if not plugin_options: - error = f"Invalid options for lemur_acme plugin: {options}" - current_app.logger.error(error) - raise InvalidConfiguration(error) - # Define static acme_root based off configuration variable by default. However, if user has passed a - # certificate, use this certificate as the root. - acme_root = current_app.config.get("ACME_ROOT") - for option in plugin_options: - if option.get("name") == "certificate": - acme_root = option.get("value") - if option.get("name") == "acme_url": - validate_acme_url(option.get("value", "")) - return acme_root, "", [role] - - def cancel_ordered_certificate(self, pending_cert, **kwargs): - # Needed to override issuer function. - pass - - def revoke_certificate(self, certificate, reason): - self.acme = AcmeHandler() - - crl_reason = CRLReason.unspecified - if "crl_reason" in reason: - crl_reason = CRLReason[reason["crl_reason"]] - - return self.acme.revoke_certificate(certificate, crl_reason.value) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py deleted file mode 100644 index ddbb94707d..0000000000 --- a/lemur/plugins/lemur_acme/powerdns.py +++ /dev/null @@ -1,438 +0,0 @@ -import json -import sys -import time - -import lemur.common.utils as utils -import lemur.dns_providers.util as dnsutil -import requests -from flask import current_app -from sentry_sdk import capture_exception - -from lemur.extensions import metrics - -REQUIRED_VARIABLES = [ - "ACME_POWERDNS_APIKEYNAME", - "ACME_POWERDNS_APIKEY", - "ACME_POWERDNS_DOMAIN", -] - - -class Zone: - """ - This class implements a PowerDNS zone in JSON. - """ - - def __init__(self, _data): - self._data = _data - - @property - def id(self): - """ Zone id, has a trailing "." at the end, which we manually remove. """ - return self._data["id"][:-1] - - @property - def name(self): - """ Zone name, has a trailing "." at the end, which we manually remove. """ - return self._data["name"][:-1] - - @property - def kind(self): - """ Indicates whether the zone is setup as a PRIMARY or SECONDARY """ - return self._data["kind"] - - -class Record: - """ - This class implements a PowerDNS record. - """ - - def __init__(self, _data): - self._data = _data - - @property - def name(self): - return self._data["name"] - - @property - def type(self): - return self._data["type"] - - @property - def ttl(self): - return self._data["ttl"] - - @property - def content(self): - return self._data["content"] - - @property - def disabled(self): - return self._data["disabled"] - - -def get_zones(account_number): - """ - Retrieve authoritative zones from the PowerDNS API and return a list of zones - - :param account_number: - :raise: Exception - :return: list of Zone Objects - """ - _check_conf() - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - path = f"/api/v1/servers/{server_id}/zones" - zones = [] - function = sys._getframe().f_code.co_name - log_data = { - "function": function - } - try: - records = _get(path) - log_data["message"] = "Retrieved Zones Successfully" - current_app.logger.debug(log_data) - - except Exception as e: - capture_exception() - log_data["message"] = "Failed to Retrieve Zone Data" - current_app.logger.debug(log_data) - raise - - for record in records: - zone = Zone(record) - if zone.kind == 'Master': - zones.append(zone.name) - return zones - - -def create_txt_record(domain, token, account_number): - """ - Create a TXT record for the given domain and token and return a change_id tuple - - :param domain: FQDN - :param token: challenge value - :param account_number: - :return: tuple of domain/token - """ - _check_conf() - - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - } - - # Create new record - domain_id = domain + "." - records = [Record({'name': domain_id, 'content': f"\"{token}\"", 'disabled': False})] - - # Get current records - cur_records = _get_txt_records(domain) - for record in cur_records: - if record.content != token: - records.append(record) - - try: - _patch_txt_records(domain, account_number, records) - log_data["message"] = "TXT record(s) successfully created" - current_app.logger.debug(log_data) - except Exception as e: - capture_exception() - log_data["Exception"] = e - log_data["message"] = "Unable to create TXT record(s)" - current_app.logger.debug(log_data) - - change_id = (domain, token) - return change_id - - -def wait_for_dns_change(change_id, account_number=None): - """ - Checks the authoritative DNS Server to see if changes have propagated. - - :param change_id: tuple of domain/token - :param account_number: - :return: - """ - _check_conf() - domain, token = change_id - number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", 3) - zone_name = _get_zone_name(domain, account_number) - nameserver = dnsutil.get_authoritative_nameserver(zone_name) - record_found = False - for attempts in range(0, number_of_attempts): - txt_records = dnsutil.get_dns_records(domain, "TXT", nameserver) - for txt_record in txt_records: - if txt_record == token: - record_found = True - break - if record_found: - break - time.sleep(10) - - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "status": record_found, - "message": "Record status on PowerDNS authoritative server" - } - current_app.logger.debug(log_data) - - if record_found: - metrics.send(f"{function}.success", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) - else: - metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) - - -def delete_txt_record(change_id, account_number, domain, token): - """ - Delete the TXT record for the given domain and token - - :param change_id: tuple of domain/token - :param account_number: - :param domain: FQDN - :param token: challenge to delete - :return: - """ - _check_conf() - - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - } - - """ - Get existing TXT records matching the domain from DNS - The token to be deleted should already exist - There may be other records with different tokens as well - """ - cur_records = _get_txt_records(domain) - found = False - new_records = [] - for record in cur_records: - if record.content == f"\"{token}\"": - found = True - else: - new_records.append(record) - - # Since the matching token is not in DNS, there is nothing to delete - if not found: - log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records" - current_app.logger.debug(log_data) - return - - # The record to delete has been found AND there are other tokens set on the same domain - # Since we only want to delete one token value from the RRSet, we need to use the Patch command to - # overwrite the current RRSet with the existing records. - elif new_records: - try: - _patch_txt_records(domain, account_number, new_records) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as e: - capture_exception() - log_data["Exception"] = e - log_data["message"] = "Unable to delete TXT record: patching exception" - current_app.logger.debug(log_data) - - # The record to delete has been found AND there are no other token values set on the same domain - # Use the Delete command to delete the whole RRSet. - else: - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "DELETE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token - } - try: - _patch(path, payload) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as e: - capture_exception() - log_data["Exception"] = e - log_data["message"] = "Unable to delete TXT record" - current_app.logger.debug(log_data) - - -def _check_conf(): - """ - Verifies required configuration variables are set - - :return: - """ - utils.validate_conf(current_app, REQUIRED_VARIABLES) - - -def _generate_header(): - """ - Generate a PowerDNS API header and return it as a dictionary - - :return: Dict of header parameters - """ - api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") - api_key = current_app.config.get("ACME_POWERDNS_APIKEY") - headers = {api_key_name: api_key} - return headers - - -def _get_zone_name(domain, account_number): - """ - Get most specific matching zone for the given domain and return as a String - - :param domain: FQDN - :param account_number: - :return: FQDN of domain - """ - zones = get_zones(account_number) - zone_name = "" - for z in zones: - if domain.endswith(z): - if z.count(".") > zone_name.count("."): - zone_name = z - if not zone_name: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "message": "No PowerDNS zone name found.", - } - metrics.send(f"{function}.fail", "counter", 1) - return zone_name - - -def _get_txt_records(domain): - """ - Retrieve TXT records for a given domain and return list of Record Objects - - :param domain: FQDN - :return: list of Record objects - """ - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - - path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" - function = sys._getframe().f_code.co_name - log_data = { - "function": function - } - try: - records = _get(path) - log_data["message"] = "Retrieved TXT Records Successfully" - current_app.logger.debug(log_data) - - except Exception as e: - capture_exception() - log_data["Exception"] = e - log_data["message"] = "Failed to Retrieve TXT Records" - current_app.logger.debug(log_data) - return [] - - txt_records = [] - for record in records: - cur_record = Record(record) - txt_records.append(cur_record) - return txt_records - - -def _get(path, params=None): - """ - Execute a GET request on the given URL (base_uri + path) and return response as JSON object - - :param path: Relative URL path - :param params: additional parameters - :return: json response - """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") - verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) - resp = requests.get( - f"{base_uri}{path}", - headers=_generate_header(), - params=params, - verify=verify_value - ) - resp.raise_for_status() - return resp.json() - - -def _patch_txt_records(domain, account_number, records): - """ - Send Patch request to PowerDNS Server - - :param domain: FQDN - :param account_number: - :param records: List of Record objects - :return: - """ - domain_id = domain + "." - - # Create records - txt_records = [] - for record in records: - txt_records.append( - {'content': record.content, 'disabled': record.disabled} - ) - - # Create RRSet - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "REPLACE", - "records": txt_records, - "comments": [] - } - ] - } - - # Create Txt Records - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_name = _get_zone_name(domain, account_number) - zone_id = zone_name + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - _patch(path, payload) - - -def _patch(path, payload): - """ - Execute a Patch request on the given URL (base_uri + path) with given payload - - :param path: - :param payload: - :return: - """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") - verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) - resp = requests.patch( - f"{base_uri}{path}", - data=json.dumps(payload), - headers=_generate_header(), - verify=verify_value - ) - resp.raise_for_status() diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py deleted file mode 100644 index f54f2a73d6..0000000000 --- a/lemur/plugins/lemur_acme/route53.py +++ /dev/null @@ -1,135 +0,0 @@ -import time - -from lemur.plugins.lemur_aws.sts import sts_client - - -@sts_client("route53") -def wait_for_dns_change(change_id, client=None): - _, change_id = change_id - - while True: - response = client.get_change(Id=change_id) - if response["ChangeInfo"]["Status"] == "INSYNC": - return - time.sleep(5) - - -@sts_client("route53") -def find_zone_id(domain, client=None): - return _find_zone_id(domain, client) - - -def _find_zone_id(domain, client=None): - paginator = client.get_paginator("list_hosted_zones") - min_diff_length = float("inf") - chosen_zone = None - - for page in paginator.paginate(): - for zone in page["HostedZones"]: - # strip the trailing "." to match against the domain (but return the full, original value) - zone_name = zone["Name"].rstrip(".") - if domain == zone_name or domain.endswith("." + zone_name): - if not zone["Config"]["PrivateZone"]: - diff_length = len(domain) - len(zone_name) - if diff_length < min_diff_length: - min_diff_length = diff_length - chosen_zone = (zone["Name"], zone["Id"]) - - if chosen_zone is None: - raise ValueError(f"Unable to find a Route53 hosted zone for {domain}") - - return chosen_zone[1] # Return the chosen zone ID - - -@sts_client("route53") -def get_zones(client=None): - paginator = client.get_paginator("list_hosted_zones") - zones = [] - for page in paginator.paginate(): - for zone in page["HostedZones"]: - if not zone["Config"]["PrivateZone"]: - zones.append( - zone["Name"][:-1] - ) # We need [:-1] to strip out the trailing dot. - return zones - - -@sts_client("route53") -def change_txt_record(action, zone_id, domain, value, client=None): - current_txt_records = [] - try: - current_records = client.list_resource_record_sets( - HostedZoneId=zone_id, - StartRecordName=domain, - StartRecordType="TXT", - MaxItems="1", - )["ResourceRecordSets"] - - for record in current_records: - if record.get("Type") == "TXT": - current_txt_records.extend(record.get("ResourceRecords", [])) - except Exception as e: - # Current Resource Record does not exist - if "NoSuchHostedZone" not in str(type(e)): - raise - # For some reason TXT records need to be - # manually quoted. - seen = False - for record in current_txt_records: - for k, v in record.items(): - if f'"{value}"' == v: - seen = True - if not seen: - current_txt_records.append({"Value": f'"{value}"'}) - - if action == "DELETE" and len(current_txt_records) > 1: - # If we want to delete one record out of many, we'll update the record to not include the deleted value instead. - # This allows us to support concurrent issuance. - current_txt_records = [ - record - for record in current_txt_records - if not (record.get("Value") == f'"{value}"') - ] - action = "UPSERT" - - response = client.change_resource_record_sets( - HostedZoneId=zone_id, - ChangeBatch={ - "Changes": [ - { - "Action": action, - "ResourceRecordSet": { - "Name": domain, - "Type": "TXT", - "TTL": 300, - "ResourceRecords": current_txt_records, - }, - } - ] - }, - ) - return response["ChangeInfo"]["Id"] - - -def create_txt_record(host, value, account_number): - zone_id = find_zone_id(host, account_number=account_number) - change_id = change_txt_record( - "UPSERT", zone_id, host, value, account_number=account_number - ) - - return zone_id, change_id - - -def delete_txt_record(change_ids, account_number, host, value): - for change_id in change_ids: - zone_id, _ = change_id - try: - change_txt_record( - "DELETE", zone_id, host, value, account_number=account_number - ) - except Exception as e: - if "but it was not found" in e.response.get("Error", {}).get("Message"): - # We tried to delete a record that doesn't exist. We'll ignore this error. - pass - else: - raise diff --git a/lemur/plugins/lemur_acme/tests/conftest.py b/lemur/plugins/lemur_acme/tests/conftest.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py deleted file mode 100644 index 7fc227d30c..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ /dev/null @@ -1,473 +0,0 @@ -import unittest -from unittest.mock import MagicMock -from unittest.mock import patch, Mock - -import josepy as jose -from acme.messages import STATUS_PENDING, STATUS_VALID -from cryptography.x509 import DNSName -from flask import Flask, current_app - -from lemur.common.utils import generate_private_key -from lemur.plugins.lemur_acme import plugin -from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord -from lemur.tests.conf import LEMUR_ENCRYPTION_KEYS - - -class TestAcmeDns(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - def setUp(self, mock_dns_provider_service): - self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeDnsHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } - - # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is - # needed to run tests in dev environment without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - def test_get_dns_challenges(self, mock_len): - assert mock_len - - from acme import challenges - - host = "example.com" - c = challenges.DNS01() - - mock_authz = Mock() - mock_authz.body.resolved_combinations = [] - mock_entry = Mock() - mock_entry.chall = c - mock_authz.body.resolved_combinations.append(mock_entry) - result, hostname_still_validated = yield self.acme.get_dns_challenges(host, mock_authz) - self.assertEqual(result, mock_entry) - self.assertFalse(hostname_still_validated) - - @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - def test_get_dns_challenges_already_valid(self, mock_len): - assert mock_len - - from acme import challenges - - host = "example.com" - c = challenges.DNS01() - - mock_authz = MagicMock() - mock_authz.body = STATUS_VALID - mock_authz.body.resolved_combinations = [] - mock_entry = Mock() - mock_entry.chall = c - mock_authz.body.resolved_combinations.append(mock_entry) - result, hostname_still_validatd = yield self.acme.get_dns_challenges(host, mock_authz) - self.assertEqual(result, mock_entry) - self.assertTrue(hostname_still_validatd) - - @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - def test_get_dns_challenges_mixed_valid(self, mock_len): - assert mock_len - - from acme import challenges - - host = "example.com" - c = challenges.DNS01() - - host2 = "www.example.com" - c2 = challenges.DNS01() - - mock_authz = MagicMock() - mock_authz.body = STATUS_VALID - mock_authz.body.resolved_combinations = [] - mock_entry = Mock() - mock_entry.chall = c - mock_authz.body.resolved_combinations.append(mock_entry) - - mock_authz2 = MagicMock() - mock_authz2.body = STATUS_PENDING - mock_authz2.body.resolved_combinations = [] - mock_entry2 = Mock() - mock_entry.chall = c2 - mock_authz.body.resolved_combinations.append(mock_entry2) - - # a pending status, mixed with valid - result, hostname_still_validatd = yield self.acme.get_dns_challenges(host2, [mock_authz, mock_authz2]) - self.assertEqual(result, mock_entry) - self.assertFalse(hostname_still_validatd) - - @patch("acme.client.ClientV2") - @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") - def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_acme - ): - assert mock_len - mock_order = Mock() - mock_authz = Mock() - mock_authz.body.resolved_combinations = [] - mock_entry = MagicMock() - - mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail - mock_authz.body.resolved_combinations.append(mock_entry) - mock_acme.request_domain_challenges = Mock(return_value=mock_authz) - mock_dns_provider = Mock() - mock_dns_provider.create_txt_record = Mock(return_value=1) - - hostname_still_validated = False - values = [mock_entry, hostname_still_validated] - iterable = mock_get_dns_challenges.return_value - iterator = iter(values) - iterable.__iter__.return_value = iterator - result = self.acme.start_dns_challenge( - mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {} - ) - self.assertEqual(type(result), AuthorizationRecord) - - @patch("acme.client.ClientV2") - @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") - @patch("time.sleep") - def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_acme - ): - mock_dns_provider = Mock() - mock_dns_provider.wait_for_dns_change = Mock(return_value=True) - mock_authz = Mock() - mock_sleep.return_value = False - mock_authz.dns_challenge.response = Mock() - mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) - mock_authz.authz = [] - mock_authz.target_domain = "www.test.com" - mock_authz_record = Mock() - mock_authz_record.body.identifier.value = "test" - mock_authz.authz.append(mock_authz_record) - mock_authz.change_id = [] - mock_authz.change_id.append("123") - mock_authz.dns_challenge = [] - dns_challenge = MagicMock() - dns_challenge["status"] == STATUS_PENDING - mock_authz.dns_challenge.append(dns_challenge) - self.acme.complete_dns_challenge(mock_acme, mock_authz) - - @patch("acme.client.ClientV2") - @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") - def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_acme - ): - mock_dns_provider = Mock() - mock_dns_provider.wait_for_dns_change = Mock(return_value=True) - - mock_dns_challenge = MagicMock() - response = Mock() - response.simple_verify = Mock(return_value=False) - mock_dns_challenge.response = Mock(return_value=response) - mock_dns_challenge["status"] == STATUS_PENDING - - mock_authz = Mock() - mock_authz.dns_challenge = [] - mock_authz.dns_challenge.append(mock_dns_challenge) - - mock_authz.target_domain = "www.test.com" - mock_authz_record = Mock() - mock_authz_record.body.identifier.value = "test" - mock_authz.authz = [] - mock_authz.authz.append(mock_authz_record) - mock_authz.change_id = [] - mock_authz.change_id.append("123") - with self.assertRaises(ValueError): - self.acme.complete_dns_challenge(mock_acme, mock_authz) - - @patch("acme.client.ClientV2") - @patch("OpenSSL.crypto", return_value="mock_cert") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") - def test_request_certificate( - self, - mock_get_dns_challenges, - mock_crypto, - mock_acme, - ): - mock_cert_response = Mock() - mock_cert_response.body = "123" - mock_cert_response_full = [mock_cert_response, True] - mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full) - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz.append(mock_authz_record) - mock_acme.fetch_chain = Mock(return_value="mock_chain") - mock_crypto.dump_certificate = Mock(return_value=b"chain") - mock_order = Mock() - self.acme.request_certificate(mock_acme, [], mock_order) - - def test_setup_acme_client_fail(self): - mock_authority = Mock() - mock_authority.options = [] - with self.assertRaises(Exception): - self.acme.setup_acme_client_no_retry(mock_authority) - - @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads") - @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2") - def test_setup_acme_client_success_load_account_from_authority(self, mock_acme, mock_key_json_load): - mock_authority = Mock() - mock_authority.id = 2 - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ - '{"name": "store_account", "value": true},' \ - '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' \ - '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]' - mock_client = Mock() - mock_acme.return_value = mock_client - - mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048")) - - result_client, result_registration = self.acme.setup_acme_client(mock_authority) - - mock_acme.new_account.assert_not_called() - assert result_client - assert not result_registration - - @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json") - @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") - @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2") - def test_setup_acme_client_success_store_new_account(self, mock_acme, mock_authorities_service, - mock_key_generation): - current_app.config["LEMUR_ENCRYPTION_KEYS"] = LEMUR_ENCRYPTION_KEYS - mock_authority = Mock() - mock_authority.id = 2 - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ - '{"name": "store_account", "value": true}]' - mock_client = Mock() - mock_registration = Mock() - mock_registration.uri = "http://test.com" - mock_client.register = mock_registration - mock_client.agree_to_tos = Mock(return_value=True) - mock_client.new_account.return_value = mock_registration - mock_acme.return_value = mock_client - - mock_key_generation.return_value = {"n": "PwIOkViO"} - - mock_authorities_service.update_options = Mock(return_value=True) - - self.acme.setup_acme_client(mock_authority) - - mock_authorities_service.update_options.assert_called_once() - assert "acme_private_key" in mock_authorities_service.update_options.call_args[1]['options'] - - @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") - @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2") - def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): - mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ - '{"name": "store_account", "value": false}]' - mock_client = Mock() - mock_registration = Mock() - mock_registration.uri = "http://test.com" - mock_client.register = mock_registration - mock_client.agree_to_tos = Mock(return_value=True) - mock_acme.return_value = mock_client - result_client, result_registration = self.acme.setup_acme_client(mock_authority) - mock_authorities_service.update_options.assert_not_called() - assert result_client - assert result_registration - - def test_get_domains_single(self): - options = {"common_name": "test.netflix.net"} - result = self.acme.get_domains(options) - self.assertEqual(result, [options["common_name"]]) - - def test_get_domains_multiple(self): - options = { - "common_name": "test.netflix.net", - "extensions": { - "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} - }, - } - result = self.acme.get_domains(options) - self.assertEqual( - result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] - ) - - def test_get_domains_san(self): - options = { - "common_name": "test.netflix.net", - "extensions": { - "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} - }, - } - result = self.acme.get_domains(options) - self.assertEqual( - result, [options["common_name"], "test2.netflix.net"] - ) - - def test_create_authority(self): - options = { - "name": "test ACME authority", - "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} - } - acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options) - self.assertEqual(acme_root, "123") - self.assertEqual(b, "") - self.assertEqual(role, [{"username": "", "password": "", "name": "acme_test_ACME_authority_admin"}]) - - @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - def test_get_dns_provider(self, mock_dns_provider_service): - provider = plugin.AcmeDnsHandler() - route53 = provider.get_dns_provider("route53") - assert route53 - cloudflare = provider.get_dns_provider("cloudflare") - assert cloudflare - dyn = provider.get_dns_provider("dyn") - assert dyn - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") - @patch("lemur.plugins.lemur_acme.challenge_types.authorization_service") - def test_create_certificate( - self, - mock_authorization_service, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_acme, - ): - provider = plugin.ACMEIssuerPlugin() - mock_authority = Mock() - mock_authority.options = '[{}]' - - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - - mock_dns_provider = Mock() - mock_dns_provider.credentials = '{"account_id": 1}' - mock_dns_provider.provider_type = "route53" - mock_dns_provider_service.get.return_value = mock_dns_provider - - issuer_options = { - "authority": mock_authority, - "dns_provider": mock_dns_provider, - "common_name": "test.netflix.net", - } - csr = "123" - mock_request_certificate.return_value = ("pem_certificate", "chain") - result = provider.create_certificate(csr, issuer_options) - assert result - - issuer_options["create_immediately"] = True - immediate_result = provider.create_certificate(csr, issuer_options) - assert immediate_result - - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", return_value="test") - def test_get_authorizations(self, mock_start_dns_challenge): - mock_order = Mock() - mock_order.body.identifiers = [] - mock_domain = Mock() - mock_order.body.identifiers.append(mock_domain) - mock_order_info = Mock() - mock_order_info.account_number = 1 - mock_order_info.domains = ["test.fakedomain.net"] - result = self.acme.get_authorizations( - "acme_client", mock_order, mock_order_info - ) - self.assertEqual(result, ["test"]) - - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge", - return_value="test", - ) - def test_finalize_authorizations(self, mock_complete_dns_challenge): - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz_record.change_id = 1 - mock_authz_record.dns_challenge.validation_domain_name = Mock() - mock_authz_record.dns_challenge.validation = Mock() - mock_authz.append(mock_authz_record) - mock_dns_provider = Mock() - mock_dns_provider.delete_txt_record = Mock() - - mock_acme_client = Mock() - result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) - self.assertEqual(result, mock_authz) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") - def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service_p, - mock_dns_provider_service, - mock_authorization_service, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificate(mock_cert) - self.assertEqual( - result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} - ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") - def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_dns_provider_service_p, - mock_authorization_service, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - mock_cert2 = Mock() - mock_cert2.external_id = 2 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificates([mock_cert, mock_cert2]) - self.assertEqual(len(result), 2) - self.assertEqual( - result[0]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, - ) - self.assertEqual( - result[1]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, - ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py deleted file mode 100644 index 692ad80c4e..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ /dev/null @@ -1,171 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -from cryptography.x509 import DNSName -from flask import Flask - -from lemur.plugins.lemur_acme import acme_handlers -from lemur.tests.vectors import ( - ACME_CHAIN_SHORT_STR, - ACME_CHAIN_LONG_STR, - SAN_CERT_STR, -) - - -class TestAcmeHandler(unittest.TestCase): - def setUp(self): - self.acme = acme_handlers.AcmeHandler() - - # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is - # needed to run tests in dev environment without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - def test_strip_wildcard(self): - expected = ("example.com", False) - result = self.acme.strip_wildcard("example.com") - self.assertEqual(expected, result) - - expected = ("example.com", True) - result = self.acme.strip_wildcard("*.example.com") - self.assertEqual(expected, result) - - def test_authz_record(self): - a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id", "cname_delegation") - self.assertEqual(type(a), acme_handlers.AuthorizationRecord) - - def test_setup_acme_client_fail(self): - mock_authority = Mock() - mock_authority.options = [] - with self.assertRaises(Exception): - self.acme.setup_acme_client_no_retry(mock_authority) - - def test_reuse_account_not_defined(self): - mock_authority = Mock() - mock_authority.options = [] - with self.assertRaises(Exception): - self.acme.reuse_account(mock_authority) - - def test_reuse_account_from_authority(self): - mock_authority = Mock() - mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]' - - self.assertTrue(self.acme.reuse_account(mock_authority)) - - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_reuse_account_from_config(self, mock_current_app): - mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' - mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"} - - self.assertTrue(self.acme.reuse_account(mock_authority)) - - def test_reuse_account_no_configuration(self): - mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' - - self.assertFalse(self.acme.reuse_account(mock_authority)) - - @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") - @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2") - def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): - mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ - '{"name": "store_account", "value": false}]' - mock_client = Mock() - mock_registration = Mock() - mock_registration.uri = "http://test.com" - mock_client.register = mock_registration - mock_client.agree_to_tos = Mock(return_value=True) - mock_acme.return_value = mock_client - result_client, result_registration = self.acme.setup_acme_client(mock_authority) - mock_authorities_service.update_options.assert_not_called() - assert result_client - assert result_registration - - def test_get_domains_single(self): - options = {"common_name": "test.netflix.net"} - result = self.acme.get_domains(options) - self.assertEqual(result, [options["common_name"]]) - - def test_get_domains_multiple(self): - options = { - "common_name": "test.netflix.net", - "extensions": { - "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} - }, - } - result = self.acme.get_domains(options) - self.assertEqual( - result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] - ) - - def test_get_domains_san(self): - options = { - "common_name": "test.netflix.net", - "extensions": { - "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} - }, - } - result = self.acme.get_domains(options) - self.assertEqual( - result, [options["common_name"], "test2.netflix.net"] - ) - - def test_extract_cert_and_chain(self): - # expecting the short chain - leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, - [ACME_CHAIN_LONG_STR], - "(STAGING) Artificial Apricot R3") - self.assertEqual(leaf_pem, SAN_CERT_STR) - self.assertEqual(chain_pem, ACME_CHAIN_SHORT_STR[len(leaf_pem):].lstrip()) - - # expecting the long chain - leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, - [ACME_CHAIN_LONG_STR], - "(STAGING) Doctored Durian Root CA X3") - self.assertEqual(leaf_pem, SAN_CERT_STR) - self.assertEqual(chain_pem, ACME_CHAIN_LONG_STR[len(leaf_pem):].lstrip()) - - -class TestPinnedClientNetwork(unittest.TestCase): - def setUp(self): - import josepy as jose - from lemur.common.utils import generate_private_key - key = jose.JWKRSA(key=generate_private_key("RSA2048")) - self.net = acme_handlers._PinnedClientNetwork( - key, - account=None, - pinned_hostname="acme-v02.api.letsencrypt.org", - ) - - def test_allows_request_to_pinned_host(self): - """Requests to the configured ACME host must be forwarded to the parent.""" - with patch.object( - acme_handlers.ClientNetwork, "_send_request", return_value=Mock() - ) as mock_send: - self.net._send_request("GET", "https://acme-v02.api.letsencrypt.org/directory") - mock_send.assert_called_once() - - def test_rejects_request_to_different_host(self): - """Requests to any host other than the pinned host must raise InvalidConfiguration.""" - from lemur.exceptions import InvalidConfiguration - with self.assertRaises(InvalidConfiguration): - self.net._send_request("POST", "https://evil.attacker.tld/steal") - - def test_rejects_internal_metadata_url(self): - """Cloud metadata endpoint must be rejected even if it slips through via ACME response.""" - from lemur.exceptions import InvalidConfiguration - with self.assertRaises(InvalidConfiguration): - self.net._send_request("POST", "http://169.254.169.254/latest/meta-data/") - - def test_rejects_internal_kubernetes_api(self): - """Internal Kubernetes API endpoint must be rejected.""" - from lemur.exceptions import InvalidConfiguration - with self.assertRaises(InvalidConfiguration): - self.net._send_request("POST", "https://kubernetes.default.svc/api/v1/secrets") diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py deleted file mode 100644 index 0e51ba9427..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ /dev/null @@ -1,188 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -from acme import challenges -from flask import Flask -from lemur.plugins.lemur_acme import plugin - -from lemur.tests.vectors import ( - ACME_CHAIN_LONG_STR, - SAN_CERT_STR, - ACME_CHAIN_X1_STR -) - - -class TestAcmeHttp(unittest.TestCase): - - def setUp(self): - self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() - self.acme = plugin.AcmeHandler() - - # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is - # needed to run tests in dev environment without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - def test_create_authority(self): - options = { - "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}, - "name": "mock_authority" - } - acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options) - self.assertEqual(acme_root, "123") - self.assertEqual(b, "") - self.assertEqual(role, [{"username": "", "password": "", "name": "acme_mock_authority_admin"}]) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.base.manager.PluginManager.get") - @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - def test_create_certificate( - self, - mock_authorization_service, - mock_request_certificate, - mock_destination_service, - mock_plugin_manager_get, - mock_acme, - ): - provider = plugin.ACMEHttpIssuerPlugin() - mock_authority = Mock() - mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' - - mock_order_resource = Mock() - mock_order_resource.authorizations = [Mock()] - mock_order_resource.authorizations[0].body.challenges = [Mock()] - mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = ( - Mock(), "Anything-goes") - mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( - token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') - - mock_client = Mock() - mock_client.new_order.return_value = mock_order_resource - mock_client.answer_challenge.return_value = True - - mock_finalized_order = Mock() - mock_finalized_order.fullchain_pem = ACME_CHAIN_LONG_STR - - mock_finalized_order.alternative_fullchains_pem = [mock_finalized_order.fullchain_pem] - mock_finalized_order.authorizations = [Mock()] - mock_client.finalize_order.return_value = mock_finalized_order - - mock_acme.return_value = (mock_client, "") - - mock_destination = Mock() - mock_destination.label = "mock-sftp-destination" - mock_destination.plugin_name = "SFTPDestinationPlugin" - mock_destination_service.get.return_value = mock_destination - - mock_destination_plugin = Mock() - mock_destination_plugin.upload_acme_token.return_value = True - mock_plugin_manager_get.return_value = mock_destination_plugin - - issuer_options = { - "authority": mock_authority, - "tokenDestination": "mock-sftp-destination", - "common_name": "test.netflix.net", - } - csr = "123" - mock_request_certificate.return_value = ("pem_certificate", "chain") - pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options) - - self.assertEqual(pem_certificate, SAN_CERT_STR) - self.assertEqual(pem_certificate_chain, ACME_CHAIN_LONG_STR[len(SAN_CERT_STR):].lstrip()) - mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"},' \ - '{"name": "drop_last_cert_from_chain", "value": true}]' - - pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options) - - self.assertEqual(pem_certificate, SAN_CERT_STR) - self.assertEqual(pem_certificate_chain, ACME_CHAIN_X1_STR) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.base.manager.PluginManager.get") - @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - def test_create_certificate_missing_destination_token( - self, - mock_authorization_service, - mock_request_certificate, - mock_destination_service, - mock_plugin_manager_get, - mock_acme, - ): - provider = plugin.ACMEHttpIssuerPlugin() - mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' - - mock_order_resource = Mock() - mock_order_resource.authorizations = [Mock()] - mock_order_resource.authorizations[0].body.challenges = [Mock()] - mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( - token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') - - mock_client = Mock() - mock_client.new_order.return_value = mock_order_resource - mock_acme.return_value = (mock_client, "") - - mock_destination = Mock() - mock_destination.label = "mock-sftp-destination" - mock_destination.plugin_name = "SFTPDestinationPlugin" - mock_destination_service.get_by_label.return_value = mock_destination - - mock_destination_plugin = Mock() - mock_destination_plugin.upload_acme_token.return_value = True - mock_plugin_manager_get.return_value = mock_destination_plugin - - issuer_options = { - "authority": mock_authority, - "tokenDestination": "mock-sftp-destination", - "common_name": "test.netflix.net", - } - csr = "123" - mock_request_certificate.return_value = ("pem_certificate", "chain") - with self.assertRaisesRegex(Exception, "No token_destination configured"): - provider.create_certificate(csr, issuer_options) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.base.manager.PluginManager.get") - @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - def test_create_certificate_missing_http_challenge( - self, - mock_authorization_service, - mock_request_certificate, - mock_destination_service, - mock_plugin_manager_get, - mock_acme, - ): - provider = plugin.ACMEHttpIssuerPlugin() - mock_authority = Mock() - mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' - - mock_order_resource = Mock() - mock_order_resource.authorizations = [Mock()] - mock_order_resource.authorizations[0].body.challenges = [Mock()] - mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01( - token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') - - mock_client = Mock() - mock_client.new_order.return_value = mock_order_resource - mock_acme.return_value = (mock_client, "") - - issuer_options = { - "authority": mock_authority, - "tokenDestination": "mock-sftp-destination", - "common_name": "test.netflix.net", - } - csr = "123" - mock_request_certificate.return_value = ("pem_certificate", "chain") - with self.assertRaisesRegex(Exception, "HTTP-01 challenge was not offered"): - provider.create_certificate(csr, issuer_options) diff --git a/lemur/plugins/lemur_acme/tests/test_nsone.py b/lemur/plugins/lemur_acme/tests/test_nsone.py deleted file mode 100644 index 8c585e1de8..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_nsone.py +++ /dev/null @@ -1,267 +0,0 @@ -""" -Unit Tests for NSone DNS provider -""" -import unittest -from unittest.mock import patch, Mock - -from flask import Flask - -from lemur.plugins.lemur_acme import plugin, nsone - - -class TestNsone(unittest.TestCase): - """ - Class for testing NSone plugin - """ - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def setUp(self, mock_dns_provider_service): - """ - unittest setup - """ - self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "nsone" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "nsone" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } - - # Creates a new Flask application for a test duration. In python 3.8, - # manual push of application context is needed to run tests in dev environment - # without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - @patch("lemur.plugins.lemur_acme.nsone.current_app") - def test_get_zones(self, mock_current_app): - """ - Testing get_zones method - """ - account_number = "1234567890" - path = "a/b/c" - zones = ['example.com', 'test.example.com'] - get_response = [{ - "dns_servers": ["string"], - "expiry": 0, - "primary_master": "string", - "id": "string", - "meta": { - "asn": ["string"], - "ca_province": ["string"], - "connections": 0, - "country": ["string"], - "georegion": ["string"], - "high_watermark": 0, - "ip_prefixes": ["string"], - "latitude": 0, - "loadAvg": 0, - "laditude": 0, - "low_watermark": 0, - "note": "string", - "priority": 0, - "pulsar": "string", - "requests": 0, - "up": True, - "us_state": ["string"], - "weight": 0 - }, - "network_pools": ["string"], - "networks": [0], - "nx_ttl": 0, - "primary": { - "enabled": True, - "secondaries": [{ - "ip": "string", - "networks": [0], - "notify": True, - "port": 0}] - }, "records": [{ - "domain": "string", - "id": "string", - "short_answers": ["string"], - "tier": 0, - "ttl": 0, - "type": "string"}], - "refresh": 0, - "retry": 0, - "ttl": 0, - "zone": "string", - "view": ["string"], - "local_tags": ["string"], - "tags": {}}] - nsone._check_conf = Mock() - nsone._get = Mock(path) - nsone._get.side_effect = [get_response] - mock_current_app.config.get = Mock(return_value="localhost") - result = nsone.get_zones(account_number) - self.assertEqual(result, zones) - - def test_get_zone_name(self): - """ - Testing get_zone_name method - """ - zones = ['example.com', 'test.example.com'] - zone = "test.example.com" - domain = "_acme-challenge.test.example.com" - account_number = "1234567890" - nsone.get_zones = Mock(return_value=zones) - result = nsone._get_zone_name(domain, account_number) - self.assertEqual(result, zone) - - @patch("lemur.plugins.lemur_acme.nsone.current_app") - def test_create_txt_record_write_only(self, mock_current_app): - """ - Testing create_txt_record without any existing data - """ - domain = "_acme_challenge.test.example.com" - zone = "example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - nsone._check_conf = Mock() - nsone._get_txt_records = Mock(return_value={"answers": []}) - nsone._get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - nsone._put = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "account": account_number, - "records": {"answers": [{"answer": [token]}]}, - "message": "TXT record(s) successfully created" - } - result = nsone.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - self.assertEqual(result, change_id) - - @patch("lemur.plugins.lemur_acme.nsone.current_app") - def test_create_txt_record_append(self, mock_current_app): - """ - Testing the create_txt_record with existing answers - """ - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - nsone._check_conf = Mock() - cur_records = { - "answers": [{ - "answer": ["ABCDEFHG"], - "id": "1234567890abcdef" - }], - "id": "1234567890abcdef"} - nsone._get_txt_records = Mock(return_value=cur_records) - nsone._get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - nsone._patch = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "account": account_number, - "records": cur_records, - "message": "TXT record(s) successfully created" - } - expected_path = "/v1/zones/test.example.com/_acme_challenge.test.example.com/TXT" - expected_payload = { - "answers": [ - {"answer": ["ABCDEFHG"], "id": "1234567890abcdef"}, - {"answer": ["ABCDEFGHIJ"]} - ], - "id": "1234567890abcdef"} - result = nsone.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - nsone._patch.assert_called_with(expected_path, expected_payload) - self.assertEqual(result, change_id) - - @patch("lemur.plugins.lemur_acme.nsone.dnsutil") - @patch("lemur.plugins.lemur_acme.nsone.current_app") - @patch("lemur.extensions.metrics") - @patch("time.sleep") - def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): - """ - Testing the wait_for_dns_change method - """ - domain = "_acme-challenge.test.example.com" - token1 = "ABCDEFG" - token2 = "HIJKLMN" - zone_name = "test.example.com" - nameserver = "1.1.1.1" - change_id = (domain, token1) - nsone._check_conf = Mock() - mock_records = (token2, token1) - mock_current_app.config.get = Mock(return_value=1) - nsone._get_zone_name = Mock(return_value=zone_name) - mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) - mock_dnsutil.get_dns_records = Mock(return_value=mock_records) - mock_sleep.return_value = False - mock_metrics.send = Mock() - mock_current_app.logger.debug = Mock() - nsone.wait_for_dns_change(change_id) - - log_data = { - "function": "wait_for_dns_change", - "fqdn": domain, - "status": True, - "account": None, - "message": "Record status on NS1 authoritative server" - } - mock_current_app.logger.debug.assert_called_with(log_data) - - @patch("lemur.plugins.lemur_acme.nsone.current_app") - def test_delete_txt_record(self, mock_current_app): - """ - Testing the delete_txt_record method. - """ - domain = "_acme-challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - records = { - "answers": [{ - "answer": ["ABCDEFG"], - "id": "1234567890abcdef" - }], - "domain": "_acme-challenge.test.example.com", - "id": "1234567890abcdef", - "meta": {}, - "networks": [1], - "regions": {}, - "tags": {}, - "tier": 1, - "ttl": 3600, - "type": "TXT", - "zone_fqdn": "example.com", - "zone_handle": "example.com", - "use_client_subnet": "false" - } - nsone._check_conf = Mock() - nsone._get_zone_name = Mock(return_value=zone) - nsone._get_txt_records = Mock(return_value=records) - nsone._delete = Mock(return_value="") - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - nsone._patch = Mock() - log_data = { - "function": "delete_txt_record", - "fqdn": domain, - "token": token, - "change": ("_acme-challenge.test.example.com", "ABCDEFGHIJ"), - "account": "1234567890", - "message": "Unable to delete TXT record: Token not found in existing TXT records" - } - nsone.delete_txt_record(change_id, account_number, domain, token) - mock_current_app.logger.debug.assert_called_with(log_data) diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py deleted file mode 100644 index cf850970a4..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ /dev/null @@ -1,183 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -from flask import Flask -from lemur.plugins.lemur_acme import plugin, powerdns - - -class TestPowerdns(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def setUp(self, mock_dns_provider_service): - self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "powerdns" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "powerdns" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } - - # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is - # needed to run tests in dev environment without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_get_zones(self, mock_current_app): - account_number = "1234567890" - path = "a/b/c" - zones = ['example.com', 'test.example.com'] - get_response = [{'account': '', 'dnssec': 'False', 'id': 'example.com.', 'kind': 'Master', 'last_check': 0, 'masters': [], - 'name': 'example.com.', 'notified_serial': '2019111907', 'serial': '2019111907', - 'url': '/api/v1/servers/localhost/zones/example.com.'}, - {'account': '', 'dnssec': 'False', 'id': 'bad.example.com.', 'kind': 'Secondary', 'last_check': 0, 'masters': [], - 'name': 'bad.example.com.', 'notified_serial': '2018053104', 'serial': '2018053104', - 'url': '/api/v1/servers/localhost/zones/bad.example.com.'}, - {'account': '', 'dnssec': 'False', 'id': 'test.example.com.', 'kind': 'Master', 'last_check': 0, - 'masters': [], 'name': 'test.example.com.', 'notified_serial': '2019112501', 'serial': '2019112501', - 'url': '/api/v1/servers/localhost/zones/test.example.com.'}] - powerdns._check_conf = Mock() - powerdns._get = Mock(path) - powerdns._get.side_effect = [get_response] - mock_current_app.config.get = Mock(return_value="localhost") - result = powerdns.get_zones(account_number) - self.assertEqual(result, zones) - - def test_get_zone_name(self): - zones = ['example.com', 'test.example.com'] - zone = "test.example.com" - domain = "_acme-challenge.test.example.com" - account_number = "1234567890" - powerdns.get_zones = Mock(return_value=zones) - result = powerdns._get_zone_name(domain, account_number) - self.assertEqual(result, zone) - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_create_txt_record_write_only(self, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - powerdns._check_conf = Mock() - powerdns._get_txt_records = Mock(return_value=[]) - powerdns._get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - powerdns._patch = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "message": "TXT record(s) successfully created" - } - result = powerdns.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - self.assertEqual(result, change_id) - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_create_txt_record_append(self, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - powerdns._check_conf = Mock() - cur_token = "123456" - cur_records = [powerdns.Record({'name': domain, 'content': f"\"{cur_token}\"", 'disabled': False})] - powerdns._get_txt_records = Mock(return_value=cur_records) - powerdns._get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - powerdns._patch = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "message": "TXT record(s) successfully created" - } - expected_path = "/api/v1/servers/localhost/zones/test.example.com." - expected_payload = { - "rrsets": [ - { - "name": domain + ".", - "type": "TXT", - "ttl": 300, - "changetype": "REPLACE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - }, - { - "content": f"\"{cur_token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } - - result = powerdns.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - powerdns._patch.assert_called_with(expected_path, expected_payload) - self.assertEqual(result, change_id) - - @patch("lemur.plugins.lemur_acme.powerdns.dnsutil") - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - @patch("lemur.extensions.metrics") - @patch("time.sleep") - def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): - domain = "_acme-challenge.test.example.com" - token1 = "ABCDEFG" - token2 = "HIJKLMN" - zone_name = "test.example.com" - nameserver = "1.1.1.1" - change_id = (domain, token1) - powerdns._check_conf = Mock() - mock_records = (token2, token1) - mock_current_app.config.get = Mock(return_value=1) - powerdns._get_zone_name = Mock(return_value=zone_name) - mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) - mock_dnsutil.get_dns_records = Mock(return_value=mock_records) - mock_sleep.return_value = False - mock_metrics.send = Mock() - mock_current_app.logger.debug = Mock() - powerdns.wait_for_dns_change(change_id) - - log_data = { - "function": "wait_for_dns_change", - "fqdn": domain, - "status": True, - "message": "Record status on PowerDNS authoritative server" - } - mock_current_app.logger.debug.assert_called_with(log_data) - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_delete_txt_record(self, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - powerdns._check_conf = Mock() - powerdns._get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - mock_current_app.config.get = Mock(return_value="localhost") - powerdns._patch = Mock() - log_data = { - "function": "delete_txt_record", - "fqdn": domain, - "token": token, - "message": "Unable to delete TXT record: Token not found in existing TXT records" - } - powerdns.delete_txt_record(change_id, account_number, domain, token) - mock_current_app.logger.debug.assert_called_with(log_data) diff --git a/lemur/plugins/lemur_acme/tests/test_route53.py b/lemur/plugins/lemur_acme/tests/test_route53.py deleted file mode 100644 index 2f7c4bbbcb..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_route53.py +++ /dev/null @@ -1,34 +0,0 @@ -from unittest.mock import MagicMock - -import pytest - -from lemur.plugins.lemur_acme.route53 import _find_zone_id -from lemur.tests.conftest import app # noqa - - -def test_zone_selection(app): # noqa - # Mocking AWS client - client = MagicMock() - - zones = [ - {"Config": {"PrivateZone": False}, "Name": "acme.identity.uq.edu.au.", "Id": "Z1"}, - {"Config": {"PrivateZone": False}, "Name": "dev.acme.identity.uq.edu.au.", "Id": "Z2"}, - {"Config": {"PrivateZone": True}, "Name": "test.dev.acme.identity.uq.edu.au.", "Id": "Z3"} - ] - - # Mocking the paginator - paginator = MagicMock() - paginator.paginate.return_value = [{"HostedZones": zones}] - client.get_paginator.return_value = paginator - - # Replace this with reference to your function - assert _find_zone_id("test.dev.acme.identity.uq.edu.au", client) == "Z2" - assert _find_zone_id("another.dev.acme.identity.uq.edu.au", client) == "Z2" - assert _find_zone_id("dev.acme.identity.uq.edu.au", client) == "Z2" - assert _find_zone_id("anotherdev.acme.identity.uq.edu.au", client) == "Z1" - assert _find_zone_id("acme.identity.uq.edu.au", client) == "Z1" - assert _find_zone_id("test2.acme.identity.uq.edu.au", client) == "Z1" - - # Test that it raises a ValueError for a domain where no matching zone is found - with pytest.raises(ValueError): - _find_zone_id("test3.some.other.domain", client) diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py deleted file mode 100644 index 7616459e7c..0000000000 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ /dev/null @@ -1,149 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -from flask import Flask -from lemur.plugins.lemur_acme import plugin, ultradns -from requests.models import Response - - -class TestUltradns(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def setUp(self, mock_dns_provider_service): - self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } - - # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is - # needed to run tests in dev environment without getting error 'Working outside of application context'. - _app = Flask('lemur_test_acme') - self.ctx = _app.app_context() - assert self.ctx - self.ctx.push() - - def tearDown(self): - self.ctx.pop() - - @patch("lemur.plugins.lemur_acme.ultradns.requests") - @patch("lemur.plugins.lemur_acme.ultradns.current_app") - def test_ultradns_get_token(self, mock_current_app, mock_requests): - # ret_val = json.dumps({"access_token": "access"}) - the_response = Response() - the_response._content = b'{"access_token": "access"}' - mock_requests.post = Mock(return_value=the_response) - mock_current_app.config.get = Mock(return_value="Test") - result = ultradns.get_ultradns_token() - self.assertTrue(len(result) > 0) - - @patch("lemur.plugins.lemur_acme.ultradns.current_app") - def test_ultradns_create_txt_record(self, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - ultradns.get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - ultradns._post = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "message": "TXT record created" - } - result = ultradns.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - self.assertEqual(result, change_id) - - @patch("lemur.plugins.lemur_acme.ultradns.current_app") - @patch("lemur.extensions.metrics") - def test_ultradns_delete_txt_record(self, mock_metrics, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - mock_current_app.logger.debug = Mock() - ultradns.get_zone_name = Mock(return_value=zone) - ultradns._post = Mock() - ultradns._get = Mock() - ultradns._get.return_value = {'zoneName': 'test.example.com.com', - 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', - 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], - 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, - 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} - ultradns._delete = Mock() - mock_metrics.send = Mock() - ultradns.delete_txt_record(change_id, account_number, domain, token) - mock_current_app.logger.debug.assert_not_called() - mock_metrics.send.assert_not_called() - - @patch("lemur.plugins.lemur_acme.ultradns.current_app") - @patch("lemur.extensions.metrics") - def test_ultradns_wait_for_dns_change(self, mock_metrics, mock_current_app): - ultradns._has_dns_propagated = Mock(return_value=True) - nameserver = "1.1.1.1" - ultradns.get_authoritative_nameserver = Mock(return_value=nameserver) - mock_metrics.send = Mock() - domain = "_acme-challenge.test.example.com" - token = "ABCDEFGHIJ" - change_id = (domain, token) - mock_current_app.logger.debug = Mock() - ultradns.wait_for_dns_change(change_id) - # mock_metrics.send.assert_not_called() - log_data = { - "function": "wait_for_dns_change", - "fqdn": domain, - "status": True, - "message": "Record status on Public DNS" - } - mock_current_app.logger.debug.assert_called_with(log_data) - - def test_ultradns_get_zone_name(self): - zones = ['example.com', 'test.example.com'] - zone = "test.example.com" - domain = "_acme-challenge.test.example.com" - account_number = "1234567890" - ultradns.get_zones = Mock(return_value=zones) - result = ultradns.get_zone_name(domain, account_number) - self.assertEqual(result, zone) - - def test_ultradns_get_zones(self): - account_number = "1234567890" - path = "a/b/c" - zones = ['example.com', 'test.example.com'] - paginate_response = [{ - 'properties': { - 'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, - 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}, { - 'properties': { - 'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, - 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}, { - 'properties': { - 'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, - 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}] - ultradns._paginate = Mock(path, "zones") - ultradns._paginate.side_effect = [[paginate_response]] - result = ultradns.get_zones(account_number) - self.assertEqual(result, zones) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py deleted file mode 100644 index 5e78d66355..0000000000 --- a/lemur/plugins/lemur_acme/ultradns.py +++ /dev/null @@ -1,447 +0,0 @@ -import time -import requests -import json -import sys - -import dns -import dns.exception -import dns.name -import dns.query -import dns.resolver - -from flask import current_app -from sentry_sdk import capture_exception - -from lemur.extensions import metrics - - -class Record: - """ - This class implements an Ultra DNS record. - - Accepts the response from the API call as the argument. - """ - - def __init__(self, _data): - # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. - # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. - self._data = _data["rrSets"][0] - - @property - def name(self): - return self._data["ownerName"] - - @property - def rrtype(self): - return self._data["rrtype"] - - @property - def rdata(self): - return self._data["rdata"] - - @property - def ttl(self): - return self._data["ttl"] - - -class Zone: - """ - This class implements an Ultra DNS zone. - """ - - def __init__(self, _data, _client="Client"): - self._data = _data - self._client = _client - - @property - def name(self): - """ - Zone name, has a trailing "." at the end, which we manually remove. - """ - return self._data["properties"]["name"][:-1] - - @property - def authoritative_type(self): - """ - Indicates whether the zone is setup as a PRIMARY or SECONDARY - """ - return self._data["properties"]["type"] - - @property - def record_count(self): - return self._data["properties"]["resourceRecordCount"] - - @property - def status(self): - """ - Returns the status of the zone - ACTIVE, SUSPENDED, etc - """ - return self._data["properties"]["status"] - - -def get_ultradns_token(): - """ - Function to call the UltraDNS Authorization API. - - Returns the Authorization access_token which is valid for 1 hour. - Each request calls this function and we generate a new token every time. - """ - path = "/v2/authorization/token" - data = { - "grant_type": "password", - "username": current_app.config.get("ACME_ULTRADNS_USERNAME", ""), - "password": current_app.config.get("ACME_ULTRADNS_PASSWORD", ""), - } - base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") - resp = requests.post(f"{base_uri}{path}", data=data, verify=True) - return resp.json()["access_token"] - - -def _generate_header(): - """ - Function to generate the header for a request. - - Contains the Authorization access_key obtained from the get_ultradns_token() function. - """ - access_token = get_ultradns_token() - return {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} - - -def _paginate(path, key): - limit = 100 - params = {"offset": 0, "limit": 1} - resp = _get(path, params) - for index in range(0, resp["resultInfo"]["totalCount"], limit): - params["offset"] = index - params["limit"] = limit - resp = _get(path, params) - yield resp[key] - - -def _get(path, params=None): - """Function to execute a GET request on the given URL (base_uri + path) with given params""" - base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") - resp = requests.get( - f"{base_uri}{path}", - headers=_generate_header(), - params=params, - verify=True, - ) - resp.raise_for_status() - return resp.json() - - -def _delete(path): - """Function to execute a DELETE request on the given URL""" - base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") - resp = requests.delete( - f"{base_uri}{path}", - headers=_generate_header(), - verify=True, - ) - resp.raise_for_status() - - -def _post(path, params): - """Executes a POST request on given URL. Body is sent in JSON format""" - base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") - resp = requests.post( - f"{base_uri}{path}", - headers=_generate_header(), - data=json.dumps(params), - verify=True, - ) - resp.raise_for_status() - - -def _has_dns_propagated(name, token, domain): - """ - Check whether the DNS change made by Lemur have propagated to the public DNS or not. - - Invoked by wait_for_dns_change() function - """ - txt_records = [] - try: - dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [domain] - dns_response = dns_resolver.query(name, "TXT") - for rdata in dns_response: - for txt_record in rdata.strings: - txt_records.append(txt_record.decode("utf-8")) - except dns.exception.DNSException: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.fail", "counter", 1) - return False - - for txt_record in txt_records: - if txt_record == token: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.success", "counter", 1) - return True - - return False - - -def wait_for_dns_change(change_id, account_number=None): - """ - Waits and checks if the DNS changes have propagated or not. - - First check the domains authoritative server. Once this succeeds, - we ask a public DNS server (Google <8.8.8.8> in our case). - """ - fqdn, token = change_id - number_of_attempts = 20 - nameserver = get_authoritative_nameserver(fqdn) - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token, nameserver) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": fqdn, - "status": status, - "message": "Record status on ultraDNS authoritative server" - } - current_app.logger.debug(log_data) - if status: - time.sleep(10) - break - time.sleep(10) - if status: - nameserver = get_public_authoritative_nameserver() - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token, nameserver) - log_data = { - "function": function, - "fqdn": fqdn, - "status": status, - "message": "Record status on Public DNS" - } - current_app.logger.debug(log_data) - if status: - metrics.send(f"{function}.success", "counter", 1) - break - time.sleep(10) - if not status: - metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token}) - capture_exception(extra={"fqdn": str(fqdn), "txt_record": str(token)}) - return - - -def get_zones(account_number): - """Get zones from the UltraDNS""" - path = "/v2/zones" - zones = [] - for page in _paginate(path, "zones"): - for elem in page: - # UltraDNS zone names end with a "." - Example - lemur.example.com. - # We pick out the names minus the "." at the end while returning the list - zone = Zone(elem) - if zone.authoritative_type == "PRIMARY" and zone.status == "ACTIVE": - zones.append(zone.name) - - return zones - - -def get_zone_name(domain, account_number): - """Get the matching zone for the given domain""" - zones = get_zones(account_number) - zone_name = "" - for z in zones: - if domain.endswith(z): - # Find the most specific zone possible for the domain - # Ex: If fqdn is a.b.c.com, there is a zone for c.com, - # and a zone for b.c.com, we want to use b.c.com. - if z.count(".") > zone_name.count("."): - zone_name = z - if not zone_name: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.fail", "counter", 1) - raise Exception(f"No UltraDNS zone found for domain: {domain}") - return zone_name - - -def create_txt_record(domain, token, account_number): - """ - Create a TXT record for the given domain. - - The part of the domain that matches with the zone becomes the zone name. - The remainder becomes the owner name (referred to as node name here) - Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we - get a request to create a cert for lemur.example.com - Domain - _acme-challenge.lemur.example.com - Matching zone - example.com - Owner name - _acme-challenge.lemur - """ - - zone_name = get_zone_name(domain, account_number) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = f"{node_name}.{zone_name}" - path = f"/v2/zones/{zone_name}/rrsets/TXT/{node_name}" - params = { - "ttl": 5, - "rdata": [ - f"{token}" - ], - } - - try: - _post(path, params) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": fqdn, - "token": token, - "message": "TXT record created" - } - current_app.logger.debug(log_data) - except Exception as e: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "domain": domain, - "token": token, - "Exception": e, - "message": "Unable to add record. Record already exists." - } - current_app.logger.debug(log_data) - - change_id = (fqdn, token) - return change_id - - -def delete_txt_record(change_id, account_number, domain, token): - """ - Delete the TXT record that was created in the create_txt_record() function. - - UltraDNS handles records differently compared to Dyn. It creates an RRSet - which is a set of records of the same type and owner. This means - that while deleting the record, we cannot delete any individual record from - the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are - being created for the same domain at the same time, the challenge TXT records - that are created will be added under the same RRSet. If the RRSet had more - than 1 record, then we create a new RRSet on UltraDNS minus the record that - has to be deleted. - """ - - if not domain: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "message": "No domain passed" - } - current_app.logger.debug(log_data) - return - - zone_name = get_zone_name(domain, account_number) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}" - - try: - rrsets = _get(path) - record = Record(rrsets) - except Exception as e: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.geterror", "counter", 1) - # No Text Records remain or host is not in the zone anymore because all records have been deleted. - return - try: - # Remove the record from the RRSet locally - record.rdata.remove(f"{token}") - except ValueError: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "token": token, - "message": "Token not found" - } - current_app.logger.debug(log_data) - return - - # Delete the RRSet from UltraDNS - _delete(path) - - # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS - if len(record.rdata) > 0: - params = { - "ttl": 5, - "rdata": record.rdata, - } - _post(path, params) - - -def delete_acme_txt_records(domain): - - if not domain: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "message": "No domain passed" - } - current_app.logger.debug(log_data) - return - acme_challenge_string = "_acme-challenge" - if not domain.startswith(acme_challenge_string): - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "domain": domain, - "acme_challenge_string": acme_challenge_string, - "message": "Domain does not start with the acme challenge string" - } - current_app.logger.debug(log_data) - return - - zone_name = get_zone_name(domain) - zone_parts = len(zone_name.split(".")) - node_name = ".".join(domain.split(".")[:-zone_parts]) - path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}" - - _delete(path) - - -def get_authoritative_nameserver(domain): - """Get the authoritative nameserver for the given domain""" - n = dns.name.from_text(domain) - - depth = 2 - default = dns.resolver.get_default_resolver() - nameserver = default.nameservers[0] - - last = False - while not last: - s = n.split(depth) - - last = s[0].to_unicode() == "@" - sub = s[1] - - query = dns.message.make_query(sub, dns.rdatatype.NS) - response = dns.query.udp(query, nameserver) - - rcode = response.rcode() - if rcode != dns.rcode.NOERROR: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.error", "counter", 1) - if rcode == dns.rcode.NXDOMAIN: - raise Exception("%s does not exist." % sub) - else: - raise Exception("Error %s" % dns.rcode.to_text(rcode)) - - if len(response.authority) > 0: - rrset = response.authority[0] - else: - rrset = response.answer[0] - - rr = rrset[0] - if rr.rdtype != dns.rdatatype.SOA: - authority = rr.target - nameserver = default.query(authority).rrset[0].to_text() - - depth += 1 - - return nameserver - - -def get_public_authoritative_nameserver(): - return "8.8.8.8" diff --git a/lemur/tests/test_authorities.py b/lemur/tests/test_authorities.py index 2207d133cd..f614075891 100644 --- a/lemur/tests/test_authorities.py +++ b/lemur/tests/test_authorities.py @@ -3,7 +3,6 @@ import pytest from lemur.authorities.views import * # noqa -from lemur.exceptions import InvalidConfiguration from lemur.tests.factories import AuthorityFactory, RoleFactory from lemur.tests.vectors import ( VALID_ADMIN_API_TOKEN, @@ -482,109 +481,6 @@ def test_authorities_put_update_options(client, authority_number, token, status) assert 'updated' in json.dumps(response[field]) -def test_update_rejects_disallowed_acme_url(app, session): - """Defect A: authority update must reject an acme_url not on the allowlist.""" - from lemur.authorities.service import update - - auth = AuthorityFactory() - session.flush() - - evil_options = json.dumps([{"name": "acme_url", "value": "https://evil.attacker.tld/dir"}]) - with pytest.raises(InvalidConfiguration): - update( - auth.id, - owner=auth.owner, - description=auth.description, - active=True, - roles=[], - options=evil_options, - ) - - -def test_update_accepts_allowlisted_acme_url(app, session): - """Defect A: authority update must accept an acme_url that is on the allowlist.""" - from lemur.authorities.service import update - - auth = AuthorityFactory() - session.flush() - - good_options = json.dumps([{"name": "acme_url", "value": "https://acme-v02.api.letsencrypt.org/directory"}]) - result = update( - auth.id, - owner=auth.owner, - description=auth.description, - active=True, - roles=[], - options=good_options, - ) - assert result is not None - - -def test_update_options_rejects_disallowed_acme_url(app, session): - """Defect A: update_options must also reject a disallowed acme_url.""" - from lemur.authorities.service import update_options - - auth = AuthorityFactory() - session.flush() - - evil_options = json.dumps([{"name": "acme_url", "value": "https://evil.attacker.tld/dir"}]) - with pytest.raises(InvalidConfiguration): - update_options(auth.id, evil_options) - - -def test_update_options_without_acme_url_is_unaffected(app, session): - """Defect A: update_options for non-ACME authorities (no acme_url) must succeed.""" - from lemur.authorities.service import update_options - - auth = AuthorityFactory() - session.flush() - - options = json.dumps([{"name": "some_other_option", "value": "foo"}]) - result = update_options(auth.id, options) - assert result is not None - - -def test_authority_service_update_rejects_disallowed_acme_url(authority, session): - from lemur.authorities import service - - options = json.dumps( - [{"name": "acme_url", "value": "http://169.254.169.254/latest/meta-data/"}] - ) - - with pytest.raises(InvalidConfiguration): - service.update( - authority.id, - description=authority.description, - owner=authority.owner, - active=authority.active, - roles=[], - options=options, - ) - - session.refresh(authority) - assert authority.options != options - - -def test_authority_service_update_allows_allowlisted_acme_url(authority, session): - from lemur.authorities import service - - options = json.dumps( - [{"name": "acme_url", "value": "https://acme-v02.api.letsencrypt.org/directory"}] - ) - - service.update( - authority.id, - description=authority.description, - owner=authority.owner, - active=authority.active, - roles=[], - options=options, - ) - - session.refresh(authority) - assert authority.options == options - - def test_authority_service_update_without_acme_url_unaffected(authority, session): from lemur.authorities import service diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 22f22e3a4e..8f2d74f0d0 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -35,16 +35,6 @@ def test_generate_private_key(): assert generate_private_key("ECCSECP384R1") assert generate_private_key("ECCSECP521R1") assert generate_private_key("ECCSECP256K1") - assert generate_private_key("ECCSECT163K1") - assert generate_private_key("ECCSECT233K1") - assert generate_private_key("ECCSECT283K1") - assert generate_private_key("ECCSECT409K1") - assert generate_private_key("ECCSECT571K1") - assert generate_private_key("ECCSECT163R2") - assert generate_private_key("ECCSECT233R1") - assert generate_private_key("ECCSECT283R1") - assert generate_private_key("ECCSECT409R1") - assert generate_private_key("ECCSECT571R2") with pytest.raises(Exception): generate_private_key("LEMUR")