Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions benchmark/bench.cc
Original file line number Diff line number Diff line change
Expand Up @@ -171,11 +171,11 @@ next_read_request(struct random_data *rdata)
} res;
random_r(rdata, &res.r[0]);
random_r(rdata, &res.r[1]);
// random file descriptor from those opened
size_t i = (res.val & (BUFSIZE - 1)) % _files.size();
// use the low order bits to pick a random file descriptor and length from those opened
size_t i = (size_t)((res.val & (BUFSIZE - 1)) % (uint64_t)(_files.size()));
pair<int, off_t> f = _files[i];
// random offset with BUFSIZE alignment
f.second = ((res.val & ~(BUFSIZE - 1)) % f.second);
// use the high order bits as a random offset with BUFSIZE alignment within the length of the file
f.second = (off_t)((res.val & ~(BUFSIZE - 1)) % (uint64_t)(f.second));
return f;
}

Expand Down
28 changes: 19 additions & 9 deletions ioqueue.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@
/* the request eventfd */
#define IOCB_RESFD(iocbp) (*(int*)&((iocbp)->aio_resfd))
/* the event closure data */
#define IOEV_DATA(ioev) (*(void**)&(ioev)->data)
#define IOEV_DATA(ioev) (*(void**)&((ioev)->data))
/* the event result code */
#define IOEV_RESULT(ioev) (*(ssize_t*)&((ioev)->res))
/* the event error code */
#define IOEV_ERROR(ioev) (*(int*)&((ioev)->res2))

/* KAIO syscalls - wrappers provided by libaio */
extern int io_setup(unsigned depth, aio_context_t *ctxp);
Expand Down Expand Up @@ -232,15 +236,16 @@ static int ioqueue_submit(unsigned int *nerr)
unsigned int i, n;
int ret;
for (i = 0, n = 0; i < _nwait;) {
ret = io_submit(_ctx, _nwait - i, _io_reqs + i);
const long requestCount = (long)(_nwait - i);
ret = io_submit(_ctx, requestCount, _io_reqs + i);
if (ret < 0) {
if (-ret == EBADF) {
/* head of the queue is bad, finish the request and continue */
ioqueue_request_finish(IOCB_DATA(_io_reqs[i]), -1, EBADF);
i ++;
} else {
/* ensure wait-queue occupies the head of the array */
memmove(_io_reqs, _io_reqs + i, (size_t)(_nwait - i));
memmove(_io_reqs, _io_reqs + i, (size_t)(requestCount));
Copy link
Copy Markdown
Owner Author

@gofish gofish May 30, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The behavior is unchanged here but it may be that the behavior is not ideal. We seem to be dropping all submitted requests when an error other than EBADF or EINVAL occurs for the zeroth file descriptor.

The other possibilities are

  • EAGAIN - this shouldn't happen if we are accurately tracking the kernel queue depth in the library
  • EFAULT - this shouldn't happen if we are initializing structures correctly in the library
  • EINVAL - this shouldn't happen if the context and requests were properly initialized
  • ENOSYS - this shouldn't happen if there is support for io_submit on the system

Some of these should probably abort(3) the program, and the rest should probably just return -1 and set errno.

_nwait -= i;
errno = -ret;
return -1;
Expand All @@ -259,13 +264,13 @@ static int ioqueue_submit(unsigned int *nerr)
}

/* fetch and process any completed requests */
int ioqueue_reap(unsigned int min)
int ioqueue_reap(unsigned int minRequestCount)
{
int ret, i;
unsigned int nerr;

/* cannot wait for more requests than have been allocated */
if (_nfree == _nreqs || min > _nreqs || (unsigned int)min > _nreqs - _nfree) {
if (_nfree == _nreqs || minRequestCount > _nreqs || minRequestCount > _nreqs - _nfree) {
errno = EINVAL;
return -1;
}
Expand All @@ -276,20 +281,25 @@ int ioqueue_reap(unsigned int min)

/* re-adjust minimum to account for EBADF-finished requests */
if (nerr > 0) {
min -= nerr;
minRequestCount -= nerr;
}

/* block for at least 'min' completion events */
/* block until at least the minimum number of requests have completed */
do {
ret = io_getevents(_ctx, min, _depth, _io_evs, NULL);
ret = io_getevents(_ctx, (long)(minRequestCount), (long)(_depth), _io_evs, NULL);
} while (ret < 0 && errno == EINTR);
if (ret < 0) {
return ret;
}

/* finish the reaped requests */
for (i = 0; i < ret; i++) {
ioqueue_request_finish(IOEV_DATA(&_io_evs[i]), _io_evs[i].res, (int)_io_evs[i].res2);
const struct io_event *const event = &_io_evs[i];
ioqueue_request_finish(
IOEV_DATA(event),
IOEV_RESULT(event),
IOEV_ERROR(event)
);
}
/* return the number of completed requests */
return ret + (int)nerr;
Expand Down