Merge pull request #157 from movabletype/multi_platform

Multi platform

Author: charsbar

README.md

@@ -10,41 +10,45 @@ Dockerfile to test MT.
 |image name|base image|Perl|PHP|MySQL|OpenSSL|End of Life|
 |-|-|-|-|-|-|-|
 |centos7|centos:7|5.16.3|*7.1.33*|*MariaDB 5.5.68*|1.0.2k|2024-06|
-|buster|debian:buster-slim|*5.28.1*|*7.3.31*|*MariaDB 10.3.39*|1.1.1n|2022-01|
 |bullseye|debian:bullseye-slim|*5.32.1*|*7.4.33*|*MariaDB 10.5.28*|1.1.1w|2024-08|
 |fedora35|fedora:35|*5.34.1*|*8.0.26*|8.0.31|1.1.1q|2022-12|
 |fedora37|fedora:37|*5.36.1*|*8.1.25*|*8.0.35*|3.0.9|2023-12|
-|fedora40|fedora:40|5.38.4|*8.2.29*|*MariaDB 10.11.11*|3.2.4|-|
+|fedora40|fedora:40|*5.38.4*|*8.2.29*|*MariaDB 10.11.11*|3.2.4|-|
 |fedora42|fedora:42|*5.40.3*|*8.4.16*|*9.5.0*|3.2.6|-|
 |fedora43|fedora:43|*5.42.0*|8.4.16|*8.4.7*|3.5.4|-|
-|cloud6 (\*1)|centos:7|*5.28.2*|*7.4.33*|*5.7.44*|1.0.2k|-|
 |cloud7 (\*1)|rockylinux/rockylinux:9|*5.38.2*|*8.3.29*|MariaDB 10.5.27|3.5.1|-|
 
-\*1 These images are not used in the MT cloud, but the well-known modules should have the same version (except for those used only in tests).
+\*1 This image is not used in the MT cloud, but the well-known modules should have the same versions (except for those used only in tests).
+
+## Environment list (multi platforms, suitable for mt-dev)
+
+|image name|base image|Perl|PHP|MySQL|OpenSSL|End of Life|
+|-|-|-|-|-|-|-|
+|noble|ubuntu:noble|5.38.2|8.3.6|8.4.7|3.0.13|-|
+|plucky|ubuntu:plucky|5.40.1|8.4.5|8.4.7|3.4.1|-|
+|questing|ubuntu:questing|5.42.0|8.4.11|8.4.7|3.5.3|-|
 
 ## Environment list (only for manual testing)
 
 |image name|base image|Perl|PHP|MySQL|OpenSSL|End of Life|
 |-|-|-|-|-|-|-|
+|buster (\*2)|debian:buster-slim|*5.28.1*|*7.3.31*|*MariaDB 10.3.39*|1.1.1n|2022-01|
 |centos6 (\*2)|centos:6|*5.10.1*|*5.3.3*|*5.1.73*|1.0.1e|2020-11|
 |centos8|centos:8|5.26.3|8.0.30|8.0.26|1.1.1k|2021-12|
-|fedora32 (\*2)|fedora:32|5.30.3|7.4.19|8.0.24|1.1.1k|2021-05|
 |fedora39 (\*2)|fedora:39|5.38.2|8.2.25|8.0.39|3.1.4|2024-11|
 |fedora41 (\*2)|fedora:41|5.40.3|8.3.27|8.4.7|3.2.6|-|
 |rawhide|fedora:rawhide|5.42.0|8.5.1|8.4.7|3.5.4|-|
 |rockylinux|rockylinux/rockylinux:9|5.32.1|8.1.34|8.0.44|3.5.1|-|
 |bookworm|debian:bookworm-slim|5.36.0|8.2.29|MariaDB 10.11.14|3.0.17|2028-06|
 |sid|debian:sid|5.40.1|8.4.16|MariaDB 11.8.5|3.5.4|-|
-|noble|ubuntu:noble|5.38.2|8.3.6|8.4.7|3.0.13|-|
 |amazonlinux|amazonlinux:2|5.16.3|7.4.33|MariaDB 5.5.68|1.0.2k|-|
-|amazonlinux2023 (\*4)|amazonlinux:2023|5.32.1|8.4.14|MariaDB 10.11.13|3.2.2|-|
+|amazonlinux2023|amazonlinux:2023|5.32.1|8.4.14|MariaDB 10.11.13|3.2.2|-|
 |postgresql|fedora:41|5.40.3|8.3.27|Postgres 16.11|3.2.6|-|
 |oracle (\*3)|oraclelinux:9-slim|5.32.1|8.3.29|MariaDB 10.5.29|3.5.1|-|
 |oracle8 (\*3)|oraclelinux:8-slim|5.26.3|8.2.30|MariaDB 10.3.39|1.1.1k|-|
 
 \*2 These images were used to test older versions of MT.
 \*3 with DBD::Oracle 1.80 + OracleInstantClient 26
-\*4 This image currently lacks php-dom, thus phpunit
 
 ## Special images
 
@@ -60,8 +64,32 @@ Dockerfile to test MT.
 $ perl bin/update_dockerfile.pl
 $ perl bin/build_all.pl (with or without --no-cache)
 $ perl bin/check_all.pl
+$ perl bin/test_workflows.pl
 $ perl bin/test_readme.pl
-$ perl bin/push_all.pl
+$ perl bin/build_all.pl --push
 
 then, make a pull request, review, and merge it to mirror the uploaded images.
 ```
+
+## Multi-platform build
+
+See https://docs.docker.com/build/building/multi-platform/ for details.
+If you do not use Docker Build Cloud, then prepare a remote environment with
+a different architecture from your local environment. You'll need to add your
+remote USER to the docker group and make sure remote dockerd is running.
+Create your builder. The following uses local amd64 and remote arm64, but
+you may do differently.
+
+```
+$ docker buildx create --name my_builder --node amd64 --platform linux/amd64
+$ docker buildx create --name my_builder --append --node arm64 --platform linux/arm64 ssh://USER@HOST
+```
+
+Then specify your builder to build. Only a few images support multi-platform build right now.
+
+```
+$ perl bin/build_all.pl --builder my_builder
+$ perl bin/build_all.pl --builder my_builder --push
+
+push_all.pl doesn't work well for multi-platform images.
+```

amazonlinux/docker-entrypoint.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -e
 
+
 mysql_install_db --user=mysql --skip-name-resolve --force >/dev/null
 
 bash -c "cd /usr; mysqld_safe --user=mysql --datadir=/var/lib/mysql &"

amazonlinux2023/docker-entrypoint.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -e
 
+
 mysql_install_db --user=mysql --skip-name-resolve --force >/dev/null
 
 bash -c "cd /usr; mysqld_safe --user=mysql --datadir=/var/lib/mysql &"

bin/build_all.pl

@@ -8,23 +8,18 @@
 use Mojo::File   qw/path/;
 
 GetOptions(
+    'builder=s'                         => \my $builder,
+    'push'                              => \my $push,
     'no_cache|no-cache'                 => \my $no_cache,
     'workers=i'                         => \my $workers,
     'errored|errored_only|errored-only' => \my $errored_only,
 );
 
 my %aliases = qw(
-    fedora  fedora32
-
     perl-5.16 centos7
-    perl-5.28 buster
-    perl-5.30 fedora32
-    perl-5.32 bullseye
-    perl-5.34 fedora35
-    perl-5.36 fedora37
-    perl-5.38 fedora39
-    perl-5.40 fedora41
-    perl-5.42 fedora43
+    perl-5.38 noble
+    perl-5.40 plucky
+    perl-5.42 questing
 
     php-7.3 buster
     php-7.4 fedora32
@@ -36,6 +31,8 @@
 );
 my %aliases_rev;
 
+my %multi_platform = map {$_ => 1} qw( noble plucky questing );
+
 while (my ($alias, $name) = each %aliases) {
     $aliases_rev{$name} ||= [];
     push @{ $aliases_rev{$name} }, $alias;
@@ -55,10 +52,34 @@
     }
     my $dockerfile = path("$name/Dockerfile")->slurp;
     my ($from)     = $dockerfile =~ /^FROM (\S+)/;
-    system("docker pull $from");
-    system("docker build $name $tags" . ($no_cache ? " --no-cache" : "") . " 2>&1 | tee log/build_$name.log");
+    system("docker pull $from") if $no_cache;
+    if ($builder && $multi_platform{$name}) {
+        system("docker buildx build --builder $builder --platform linux/amd64,linux/arm64 $name $tags --output=type=image" . ($no_cache ? " --no-cache" : "") . ($push ? " --push" : "") . " 2>&1 | tee log/build_$name.log");
+    } else {
+        if ($push) {
+            if ($multi_platform{$name}) {
+                say "$name requires multi-platform build to push";
+            } else {
+                my $id = `docker images movabletype/test:$name --no-trunc --format '{{.ID}}'`;
+                chomp $id;
+                die "Image not found: $name" unless $id;
+
+                say "$name : $id";
+
+                my @tags = map { s/ .*//r }
+                    grep { /$id/ }
+                    split /\n/, `docker images -f 'reference=movabletype/test:*' --no-trunc --format '{{.Tag}} {{.ID}}'`;
+
+                for my $tag (@tags) {
+                    system("docker push movabletype/test:$tag");
+                }
+            }
+        } else {
+            system("docker build $name $tags" . ($no_cache ? " --no-cache" : "") . " 2>&1 | tee log/build_$name.log");
+        }
+    }
     my $log = path("log/build_$name.log")->slurp;
-    if ($log =~ m!(naming to docker.io/movabletype/test:$name (\S+ )?done|Successfully built)!) {
+    if ($log =~ m!(naming to docker.io/movabletype/test:$name (\S+ )?done|Successfully built|exporting manifest list .* done)!) {
         if ($log =~ /No package (.+) available/) {
             rename "log/build_$name.log" => "log/build_warn_$name.log";
         } else {

bin/check_all.pl

@@ -26,7 +26,7 @@
     next if (my $conf = $dockerfile->slurp) =~ /EXPOSE/;
     my $id = substr(md5_hex($conf), 0, 7);
     diag "testing $name";
-    my $res = eval { !system("docker run -it --rm -v\$PWD:/mt -w /mt --entrypoint '' movabletype/test:$name bash -c 'TEST_IMAGE=$name prove -lv bin/checker.t' 2>&1 | tee log/check_$name.log"); };
+    my $res = eval { !system("docker run -it --rm -v\$PWD:/mt -w /mt --entrypoint '' movabletype/test:$name bash -c 'source ~/.bash_profile; TEST_IMAGE=$name prove -lv bin/checker.t' 2>&1 | tee log/check_$name.log"); };
     my ($has_ok, $has_fail, $has_todo) = (0, 0, 0);
     if ($res) {
         my $log = path("log/check_$name.log")->slurp;
@@ -78,6 +78,6 @@
     $message = colored(['red'], $message) if $fail || !$ok;
     diag $message;
 }
-diag $_ for @errors;
+diag colored([/# TODO/ ? 'yellow' : 'red'], $_) for @errors;
 
 system('docker system prune -f');

bin/checker.t

@@ -36,7 +36,7 @@ ok !grep(/\.(?:cpanm|perl-cpm)/, @files), "$image_name: no cpanm|cpm directories
 my $entrypoint_is_executed;
 if (-e '/docker-entrypoint.sh' && $image_name !~ /chromedriver/) {
     my $entrypoint_sh = do { open my $fh, '<', '/docker-entrypoint.sh'; local $/; <$fh> };
-    if ($entrypoint_sh =~ /mysql/ && $image_name !~ /(?:fedora23|postgresql)/) {
+    if ($entrypoint_sh =~ /mysql/ && $image_name !~ /(?:postgresql)/) {
         my $entrypoint = `/docker-entrypoint.sh`;
         note $entrypoint;
         $entrypoint_is_executed = 1;
@@ -116,7 +116,7 @@ ok $imagemagick_supports{png},  "$image_name: ImageMagick supports PNG";
 ok $imagemagick_supports{jpeg}, "$image_name: ImageMagick supports JPEG";
 ok $imagemagick_supports{bmp},  "$image_name: ImageMagick supports BMP";
 SKIP: {
-    local $TODO = 'WebP may not be supported' if $image_name =~ /^(?:amazonlinux|bionic|centos6|centos7|jessie|oracle|stretch|trusty)$/;
+    local $TODO = 'WebP may not be supported' if $image_name =~ /^(?:amazonlinux|centos7|oracle)$/;
     ok $imagemagick_supports{webp}, "$image_name: ImageMagick supports WebP";
 }
 SKIP: {
@@ -131,19 +131,13 @@ ok $graphicsmagick_supports{gif},  "$image_name: GraphicsMagick supports GIF";
 ok $graphicsmagick_supports{png},  "$image_name: GraphicsMagick supports PNG";
 ok $graphicsmagick_supports{jpeg}, "$image_name: GraphicsMagick supports JPEG";
 ok $graphicsmagick_supports{bmp},  "$image_name: GraphicsMagick supports BMP";
-SKIP: {
-    local $TODO = 'WebP may not be supported' if $image_name =~ /centos6|jessie|trusty/;
-    ok $graphicsmagick_supports{webp}, "$image_name: GraphicsMagick supports WebP";
-}
+ok $graphicsmagick_supports{webp}, "$image_name: GraphicsMagick supports WebP";
 SKIP: {
     local $TODO = 'AVIF may not be supported';
     ok $graphicsmagick_supports{avif}, "$image_name: GraphicsMagick supports AVIF";
 }
-SKIP: {
-    local $TODO = 'may be 8' if $image_name =~ /centos6|jessie|trusty/;
-    my $graphicsmagick_depth = Graphics::Magick->new->Get('depth');
-    is $graphicsmagick_depth => '16', "$image_name: GraphicsMagick Quantum Depth: Q$graphicsmagick_depth";
-}
+my $graphicsmagick_depth = Graphics::Magick->new->Get('depth');
+is $graphicsmagick_depth => '16', "$image_name: GraphicsMagick Quantum Depth: Q$graphicsmagick_depth";
 my ($has_identify) = `which identify`;
 ok $has_identify, "has identify";
 my ($has_convert) = `which convert`;
@@ -167,18 +161,12 @@ if ($image_name eq 'postgresql') {
     ok $phpinfo =~ /PDO drivers => .*?mysql/, "$image_name: PHP has PDO mysql driver";
 }
 ok $phpinfo =~ /GD Support => enabled/, "$image_name: PHP has GD";
-SKIP: {
-    local $TODO = 'php for CentOS6 does not support DOM/XML' if $image_name =~ /centos6/;
-    ok $phpinfo =~ /DOM.XML => enabled/, "$image_name: PHP has DOM/XML";
-}
+ok $phpinfo =~ /DOM.XML => enabled/, "$image_name: PHP has DOM/XML";
 ok $phpinfo =~ /GIF Read Support => enabled/,   "$image_name: PHP supports GIF read";
 ok $phpinfo =~ /GIF Create Support => enabled/, "$image_name: PHP supports GIF create";
 ok $phpinfo =~ /JPEG Support => enabled/,       "$image_name: PHP supports JPEG";
 ok $phpinfo =~ /PNG Support => enabled/,        "$image_name: PHP supports PNG";
-SKIP: {
-    local $TODO = 'php for CentOS6 does not support WebP' if $image_name =~ /centos6/;
-    ok $phpinfo =~ /WebP Support => enabled/, "$image_name: PHP supports WebP";
-}
+ok $phpinfo =~ /WebP Support => enabled/, "$image_name: PHP supports WebP";
 SKIP: {
     local $TODO = 'Memcache may not be supported' if $image_name =~ /amazonlinux|oracle|centos8/;
     ok $phpinfo =~ /memcache support => enabled/, "$image_name: PHP supports memcache";
@@ -284,11 +272,8 @@ ok $mailpit, "$image_name: has mailpit $mailpit";
 
 my (@icc_profiles) = (`find /usr/share | grep '.icc\$'` // '') =~ /(\w+\.icc)$/gm;
 my $srgb = grep /\bsRGB\.icc$/i, @icc_profiles;
-SKIP: {
-    local $TODO = 'CentOS 6 has no icc profile packages' if $image_name =~ /centos6/;
-    ok @icc_profiles, "$image_name: has " . join(",", @icc_profiles);
-    ok $srgb,         "$image_name: has sRGB.icc";
-}
+ok @icc_profiles, "$image_name: has " . join(",", @icc_profiles);
+ok $srgb,         "$image_name: has sRGB.icc";
 
 if ($image_name =~ /oracle/) {
     my ($sqlplus_version) = (`sqlplus -v`) =~ /^Version (\d+\.\d+)/m;

bin/push_all.pl

@@ -2,10 +2,23 @@
 use warnings;
 use Test::More;
 use Mojo::File qw/path/;
-use Data::Dump qw/dump/;
+use YAML;
 
 my %mapping;
+my $phase;
 for my $line (split /\n/, path('README.md')->slurp) {
+    if ($line =~ /^## .*for CI/) {
+        $phase = 'ci';
+    }
+    if ($line =~ /^## .*multi platforms/) {
+        $phase = 'multi';
+    }
+    if ($line =~ /^## .*manual testing/) {
+        $phase = 'manual';
+    }
+    if ($line =~ /^## Special images/) {
+        $phase = 'special';
+    }
     next unless $line =~ /^\|/;
     next if $line     =~ /^\|(?:\-|image name)/;
     $line             =~ s/(^\|)|(\|$)//g;
@@ -18,12 +31,19 @@
         $mapping{$image} = { map { split / /, $_ } @extra };
     } else {
         my ($perl, $php, $mysql, $openssl) = @rest;
+        my $ci = $phase eq 'ci' ? 1 : $image =~ /\(\\2\)/ ? 2 : 0;
         $image =~ s/ .+$//;
         $mapping{$image} = { perl => $perl, php => $php, mysql => $mysql, openssl => $openssl };
+        $mapping{$image}{ci} = $ci if $ci;
     }
     $mapping{$image}{base} = $base;
 }
 
+my $used = {};
+if (-e './tmp/used_images.yml') {
+    $used = YAML::LoadFile('./tmp/used_images.yml');
+}
+
 for my $image (sort keys %mapping) {
     my $logfile = "log/check_$image.log";
     if (!-f $logfile) {
@@ -41,11 +61,26 @@
             is $from => $mapping{$image}{$key} => "$image has correct $key";
             next;
         }
+        if ($key eq 'ci') {
+            if ($mapping{$image}{ci} == 1) {
+                ok $used->{$image}{develop}, "$image is used in workflow files";
+            }
+            if ($mapping{$image}{ci} == 2) {
+                ok $used->{$image} && !$used->{$image}{develop}, "$image is previously used in workflow files";
+            }
+            next;
+        }
         my $wanted = $key;
         $wanted = '(?:mysql|mariadb|postgresql)' if $key eq 'mysql';
         my ($version) = $log =~ /$wanted exists \((.+?)\)/i;
         is $version => $mapping{$image}{$key} => "$image has correct $key";
     }
 }
 
+for my $image (sort keys %$used) {
+    next if $mapping{$image}{ci};
+    next if $image =~ /addons8|chromiumdriver/;
+    fail "$image is no longer used in workflow files";
+}
+
 done_testing;