Skip to content

Commit af43ee2

Browse files
bukkadevnexen
bukka
and
devnexen
committed
IO copy API for stream copying
This introduces new API for fd copying and modifies php_stream_copy_to_stream_ex to use it. The implementation is separated for various platforms and the end result have couple of implications: - sendfile is used for copying file to generic fd (e.g. sockets) on all platforms except Windows that use TransmitFile - splice is used for copying between generic fds (e.g. sockets) on Linux - copy_file_range should get used on alpine linux with directly using syscall (as musl does not seem to implement it) - copy_file_range is used in the loop so it is used multiple times for files bigger than 2GB on Linux. - file mmap for copying is removed as it allowed crashing PHP when another process modified mapped file - this was used as a fallback for file copying. Sendfile should partially replace it. - File to file copying was optimized on Windows with use of ReadFile and WriteFile. This also adds various tests including Linux unit tests. Closes GH-20399 Co-authored-by: David Carlier <devnexen@gmail.com>
1 parent dcfa40c commit af43ee2

46 files changed

Lines changed: 2455 additions & 191 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/unit-tests.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@ on:
33
push:
44
paths:
55
- 'main/network.c'
6+
- 'main/io/**'
7+
- 'main/php_io.h'
68
- 'tests/unit/**'
79
- '.github/workflows/unit-tests.yml'
810
branches:
911
- master
1012
pull_request:
1113
paths:
1214
- 'main/network.c'
15+
- 'main/io/**'
16+
- 'main/php_io.h'
1317
- 'tests/unit/**'
1418
- '.github/workflows/unit-tests.yml'
1519
branches:

‎configure.ac‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,9 @@ AC_SEARCH_LIBS([Pgrab], [proc])
369369
dnl Haiku does not have network api in libc.
370370
AC_SEARCH_LIBS([setsockopt], [network])
371371

372+
dnl Solaris/illumos provide sendfile() in libsendfile; libc on Linux/FreeBSD.
373+
AC_SEARCH_LIBS([sendfile], [sendfile])
374+
372375
dnl Check for openpty. It may require linking against libutil or libbsd.
373376
AC_CHECK_FUNCS([openpty],,
374377
[AC_SEARCH_LIBS([openpty], [util bsd], [AC_DEFINE([HAVE_OPENPTY], [1])])])
@@ -588,10 +591,12 @@ AC_CHECK_FUNCS(m4_normalize([
588591
putenv
589592
reallocarray
590593
scandir
594+
sendfile
591595
setenv
592596
setitimer
593597
shutdown
594598
sigprocmask
599+
splice
595600
statfs
596601
statvfs
597602
std_syslog
@@ -1685,6 +1690,15 @@ PHP_ADD_SOURCES_X([main],
16851690
[PHP_FASTCGI_OBJS],
16861691
[no])
16871692

1693+
PHP_ADD_SOURCES([main/io], m4_normalize([
1694+
php_io.c
1695+
php_io_copy_linux.c
1696+
php_io_copy_freebsd.c
1697+
php_io_copy_solaris.c
1698+
php_io_copy_macos.c
1699+
]),
1700+
[-DZEND_ENABLE_STATIC_TSRMLS_CACHE=1])
1701+
16881702
PHP_ADD_SOURCES([main/poll], m4_normalize([
16891703
poll_backend_epoll.c
16901704
poll_backend_eventport.c

‎ext/openssl/tests/stream_copy_to_stream_ssl_to_file.phpt‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
--TEST--
2+
stream_copy_to_stream() from a TLS stream copies decrypted data (no fd fast-path)
3+
--EXTENSIONS--
4+
openssl
5+
--SKIPIF--
6+
<?php
7+
if (!function_exists("proc_open")) die("skip no proc_open");
8+
?>
9+
--FILE--
10+
<?php
11+
12+
$certFile = __DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl.pem.tmp';
13+
$cacertFile = __DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl-ca.pem.tmp';
14+
15+
$serverCode = <<<'CODE'
16+
$serverCtx = stream_context_create(['ssl' => [
17+
'local_cert' => '%s',
18+
]]);
19+
$flags = STREAM_SERVER_BIND | STREAM_SERVER_LISTEN;
20+
$server = stream_socket_server("ssl://127.0.0.1:0", $errno, $errstr, $flags, $serverCtx);
21+
phpt_notify_server_start($server);
22+
23+
$conn = stream_socket_accept($server, 5);
24+
fwrite($conn, str_repeat("secret-", 1000));
25+
fclose($conn);
26+
fclose($server);
27+
CODE;
28+
$serverCode = sprintf($serverCode, $certFile);
29+
30+
$peerName = 'stream_copy_ssl_peer';
31+
$clientCode = <<<'CODE'
32+
$clientCtx = stream_context_create(['ssl' => [
33+
'verify_peer' => true,
34+
'cafile' => '%s',
35+
'peer_name' => '%s',
36+
]]);
37+
$client = stream_socket_client("ssl://{{ ADDR }}", $errno, $errstr, 5, STREAM_CLIENT_CONNECT, $clientCtx);
38+
39+
$tmp = tmpfile();
40+
/* If the copy offloaded the raw socket fd it would write ciphertext; the
41+
* decrypted plaintext proves it correctly fell back to the userspace loop. */
42+
$copied = stream_copy_to_stream($client, $tmp);
43+
var_dump($copied);
44+
45+
fseek($tmp, 0, SEEK_SET);
46+
$content = stream_get_contents($tmp);
47+
var_dump(strlen($content));
48+
var_dump($content === str_repeat("secret-", 1000));
49+
50+
fclose($tmp);
51+
fclose($client);
52+
CODE;
53+
$clientCode = sprintf($clientCode, $cacertFile, $peerName);
54+
55+
include 'CertificateGenerator.inc';
56+
$certificateGenerator = new CertificateGenerator();
57+
$certificateGenerator->saveCaCert($cacertFile);
58+
$certificateGenerator->saveNewCertAsFileWithKey($peerName, $certFile);
59+
60+
include 'ServerClientTestCase.inc';
61+
ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
62+
?>
63+
--CLEAN--
64+
<?php
65+
@unlink(__DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl.pem.tmp');
66+
@unlink(__DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl-ca.pem.tmp');
67+
?>
68+
--EXPECT--
69+
int(7000)
70+
int(7000)
71+
bool(true)

‎ext/openssl/xp_ssl.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#include "zend_exceptions.h"
2828
#include "php_openssl.h"
2929
#include "php_openssl_backend.h"
30-
#include "php_network.h"
30+
#include "php_io.h"
3131
#include <openssl/ssl.h>
3232
#include <openssl/rsa.h>
3333
#include <openssl/x509.h>
@@ -3627,6 +3627,18 @@ static int php_openssl_sockop_cast(php_stream *stream, int castas, void **ret)
36273627
*(php_socket_t *)ret = sslsock->s.socket;
36283628
}
36293629
return SUCCESS;
3630+
case PHP_STREAM_AS_FD_FOR_COPY:
3631+
if (sslsock->ssl_active) {
3632+
return FAILURE;
3633+
}
3634+
if (ret) {
3635+
php_io_fd *copy_fd = (php_io_fd *) ret;
3636+
copy_fd->socket = sslsock->s.socket;
3637+
copy_fd->fd_type = PHP_IO_FD_SOCKET;
3638+
copy_fd->timeout = sslsock->s.timeout;
3639+
copy_fd->is_blocked = sslsock->s.is_blocked;
3640+
}
3641+
return SUCCESS;
36303642
default:
36313643
return FAILURE;
36323644
}

‎ext/standard/tests/streams/stream_copy_to_stream_file_to_file_append.phpt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
--TEST--
2+
stream_copy_to_stream() file to file with an append-mode destination
3+
--FILE--
4+
<?php
5+
6+
$srcFile = __DIR__ . '/stream_copy_append_src.txt';
7+
$dstFile = __DIR__ . '/stream_copy_append_dst.txt';
8+
9+
file_put_contents($srcFile, str_repeat("b", 3000));
10+
file_put_contents($dstFile, "PREFIX-");
11+
12+
$src = fopen($srcFile, 'r');
13+
/* O_APPEND must disable the fd-level copy fast-path and still append correctly. */
14+
$dst = fopen($dstFile, 'a');
15+
16+
$copied = stream_copy_to_stream($src, $dst);
17+
var_dump($copied);
18+
19+
fclose($src);
20+
fclose($dst);
21+
22+
$result = file_get_contents($dstFile);
23+
var_dump(strlen($result));
24+
var_dump($result === "PREFIX-" . str_repeat("b", 3000));
25+
?>
26+
--CLEAN--
27+
<?php
28+
@unlink(__DIR__ . '/stream_copy_append_src.txt');
29+
@unlink(__DIR__ . '/stream_copy_append_dst.txt');
30+
?>
31+
--EXPECT--
32+
int(3000)
33+
int(3007)
34+
bool(true)

‎ext/standard/tests/streams/stream_copy_to_stream_file_to_file_dest_read_ahead.phpt‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
--TEST--
2+
stream_copy_to_stream() file to file with a partially read destination
3+
--FILE--
4+
<?php
5+
6+
$srcFile = __DIR__ . '/stream_copy_dest_read_ahead_src.txt';
7+
$dstFile = __DIR__ . '/stream_copy_dest_read_ahead_dst.txt';
8+
9+
file_put_contents($srcFile, str_repeat("N", 50));
10+
file_put_contents($dstFile, str_repeat("O", 3000));
11+
12+
$src = fopen($srcFile, 'r');
13+
$dst = fopen($dstFile, 'r+');
14+
/* Buffered read-ahead moves the fd offset past the stream position; the copy
15+
* must land at the stream position. */
16+
fread($dst, 10);
17+
18+
$copied = stream_copy_to_stream($src, $dst);
19+
var_dump($copied);
20+
var_dump(ftell($dst));
21+
22+
fclose($src);
23+
fclose($dst);
24+
25+
$result = file_get_contents($dstFile);
26+
var_dump(strlen($result));
27+
var_dump($result === str_repeat("O", 10) . str_repeat("N", 50) . str_repeat("O", 2940));
28+
?>
29+
--CLEAN--
30+
<?php
31+
@unlink(__DIR__ . '/stream_copy_dest_read_ahead_src.txt');
32+
@unlink(__DIR__ . '/stream_copy_dest_read_ahead_dst.txt');
33+
?>
34+
--EXPECT--
35+
int(50)
36+
int(60)
37+
int(3000)
38+
bool(true)

‎ext/standard/tests/streams/stream_copy_to_stream_file_to_file_over_2gb.phpt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
--TEST--
2+
stream_copy_to_stream() copies files larger than 2GB in full
3+
--SKIPIF--
4+
<?php
5+
if (!getenv('RUN_RESOURCE_HEAVY_TESTS')) die('skip resource-heavy test');
6+
if (PHP_INT_SIZE < 8) die('skip 64-bit only');
7+
if (getenv('SKIP_SLOW_TESTS')) die('skip slow test');
8+
$dir = sys_get_temp_dir();
9+
if (disk_free_space($dir) < 4 * 1024 * 1024 * 1024) {
10+
die('skip Reason: Insufficient disk space (less than 4GB)');
11+
}
12+
?>
13+
--FILE--
14+
<?php
15+
$size = 3 * 1024 * 1024 * 1024; // exceeds the ~2GB per-call kernel copy limit
16+
$src = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_src.bin";
17+
$dst = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_dst.bin";
18+
19+
// Create a sparse 3GB source so the copy loop runs without using 3GB of data.
20+
$fh = fopen($src, "wb");
21+
fseek($fh, $size - 1);
22+
fwrite($fh, "\0");
23+
fclose($fh);
24+
25+
$in = fopen($src, "rb");
26+
$out = fopen($dst, "wb");
27+
$copied = stream_copy_to_stream($in, $out);
28+
fclose($in);
29+
fclose($out);
30+
31+
var_dump($copied === $size);
32+
var_dump(filesize($dst) === $size);
33+
34+
unlink($src);
35+
unlink($dst);
36+
?>
37+
--EXPECT--
38+
bool(true)
39+
bool(true)
40+
--CLEAN--
41+
<?php
42+
@unlink(sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_src.bin");
43+
@unlink(sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_dst.bin");
44+
?>

‎ext/standard/tests/streams/stream_copy_to_stream_file_to_socket_maxlen.phpt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
--TEST--
2+
stream_copy_to_stream() file to socket with a maxlength shorter than the file (bounded sendfile + source offset)
3+
--SKIPIF--
4+
<?php
5+
if (!function_exists("proc_open")) die("skip no proc_open");
6+
?>
7+
--FILE--
8+
<?php
9+
10+
$serverCode = <<<'CODE'
11+
$server = stream_socket_server("tcp://127.0.0.1:0", $errno, $errstr);
12+
phpt_notify_server_start($server);
13+
14+
$conn = stream_socket_accept($server);
15+
$result = stream_get_contents($conn);
16+
17+
phpt_notify(message: strlen($result));
18+
phpt_notify(message: $result === str_repeat("A", 8192) ? "match" : "mismatch");
19+
20+
fclose($conn);
21+
fclose($server);
22+
CODE;
23+
24+
$clientCode = <<<'CODE'
25+
$src = tmpfile();
26+
fwrite($src, str_repeat("A", 8192) . str_repeat("B", 8192));
27+
rewind($src);
28+
29+
$dest = stream_socket_client("tcp://{{ ADDR }}", $errno, $errstr, 10);
30+
31+
/* Only the first 8192 bytes must be sent: the bounded sendfile path has to
32+
* stop at maxlen rather than streaming to EOF. */
33+
$copied = stream_copy_to_stream($src, $dest, 8192);
34+
var_dump($copied);
35+
36+
/* The source position must have advanced by exactly maxlen, so the kernel
37+
* offload restored the descriptor offset to the maxlen boundary. */
38+
$rest = fread($src, 8192);
39+
var_dump(strlen($rest));
40+
var_dump($rest === str_repeat("B", 8192));
41+
42+
fclose($dest);
43+
fclose($src);
44+
45+
var_dump((int) trim(phpt_wait()));
46+
var_dump(trim(phpt_wait()) === "match");
47+
CODE;
48+
49+
include sprintf("%s/../../../openssl/tests/ServerClientTestCase.inc", __DIR__);
50+
ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
51+
?>
52+
--EXPECT--
53+
int(8192)
54+
int(8192)
55+
bool(true)
56+
int(8192)
57+
bool(true)

‎ext/standard/tests/streams/stream_copy_to_stream_file_to_socket_medium.phpt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
--TEST--
2+
stream_copy_to_stream() 16k with file as $source and socket as $dest
3+
--SKIPIF--
4+
<?php
5+
if (!function_exists("proc_open")) die("skip no proc_open");
6+
?>
7+
--FILE--
8+
<?php
9+
10+
$serverCode = <<<'CODE'
11+
$server = stream_socket_server("tcp://127.0.0.1:0", $errno, $errstr);
12+
phpt_notify_server_start($server);
13+
14+
$conn = stream_socket_accept($server);
15+
$data = str_repeat('data', 4096);
16+
$result = stream_get_contents($conn);
17+
18+
phpt_notify(message: strlen($result));
19+
phpt_notify(message: $result === $data ? "match" : "mismatch");
20+
21+
fclose($conn);
22+
fclose($server);
23+
CODE;
24+
25+
$clientCode = <<<'CODE'
26+
$src = tmpfile();
27+
$data = str_repeat('data', 4096);
28+
fwrite($src, $data);
29+
rewind($src);
30+
31+
$dest = stream_socket_client("tcp://{{ ADDR }}", $errno, $errstr, 10);
32+
$copied = stream_copy_to_stream($src, $dest);
33+
var_dump($copied);
34+
35+
fclose($dest);
36+
fclose($src);
37+
38+
var_dump((int) trim(phpt_wait()));
39+
var_dump(trim(phpt_wait()) === "match");
40+
CODE;
41+
42+
include sprintf("%s/../../../openssl/tests/ServerClientTestCase.inc", __DIR__);
43+
ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
44+
?>
45+
--EXPECT--
46+
int(16384)
47+
int(16384)
48+
bool(true)

0 commit comments

Comments
 (0)