[ceph.git] / ceph / src / seastar / include / seastar / core / linux-aio.hh

/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2017 ScyllaDB
 */

#pragma once

#include <endian.h>
#include <sys/time.h>
#include <sys/uio.h>
#include <signal.h>
#include <cstdint>

namespace seastar {

namespace internal {

namespace linux_abi {

using aio_context_t = unsigned long;

enum class iocb_cmd : uint16_t {
    PREAD = 0,
    PWRITE = 1,
    FSYNC = 2,
    FDSYNC = 3,
    POLL = 5,
    NOOP = 6,
    PREADV = 7,
    PWRITEV = 8,
};

struct io_event {
    uint64_t data;
    uint64_t obj;
    int64_t res;
    int64_t res2;
};

constexpr int IOCB_FLAG_RESFD = 1;

struct iocb {
        uint64_t   aio_data;

#if __BYTE_ORDER == __LITTLE_ENDIAN
        uint32_t   aio_key;
        int32_t aio_rw_flags;
#elif __BYTE_ORDER == __BIG_ENDIAN
        int32_t aio_rw_flags;
        uint32_t   aio_key;
#else
#error bad byteorder
#endif

        iocb_cmd   aio_lio_opcode;
        int16_t   aio_reqprio;
        uint32_t   aio_fildes;

        uint64_t   aio_buf;
        uint64_t   aio_nbytes;
        int64_t   aio_offset;

        uint64_t   aio_reserved2;

        uint32_t   aio_flags;

        uint32_t   aio_resfd;
};

struct aio_sigset {
    const sigset_t *sigmask;
    size_t sigsetsize;
};

}

linux_abi::iocb make_read_iocb(int fd, uint64_t offset, void* buffer, size_t len);
linux_abi::iocb make_write_iocb(int fd, uint64_t offset, const void* buffer, size_t len);
linux_abi::iocb make_readv_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov);
linux_abi::iocb make_writev_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov);
linux_abi::iocb make_poll_iocb(int fd, uint32_t events);

void set_user_data(linux_abi::iocb& iocb, void* data);
void set_nowait(linux_abi::iocb& iocb, bool nowait);

void set_eventfd_notification(linux_abi::iocb& iocb, int eventfd);

linux_abi::iocb* get_iocb(const linux_abi::io_event& ioev);

int io_setup(int nr_events, linux_abi::aio_context_t* io_context);
int io_destroy(linux_abi::aio_context_t io_context) noexcept;
int io_submit(linux_abi::aio_context_t io_context, long nr, linux_abi::iocb** iocbs);
int io_cancel(linux_abi::aio_context_t io_context, linux_abi::iocb* iocb, linux_abi::io_event* result);
int io_getevents(linux_abi::aio_context_t io_context, long min_nr, long nr, linux_abi::io_event* events, const ::timespec* timeout,
        bool force_syscall = false);
int io_pgetevents(linux_abi::aio_context_t io_context, long min_nr, long nr, linux_abi::io_event* events, const ::timespec* timeout, const sigset_t* sigmask,
        bool force_syscall = false);

void setup_aio_context(size_t nr, linux_abi::aio_context_t* io_context);

}

extern bool aio_nowait_supported;

namespace internal {

inline
linux_abi::iocb
make_read_iocb(int fd, uint64_t offset, void* buffer, size_t len) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::PREAD;
    iocb.aio_fildes = fd;
    iocb.aio_offset = offset;
    iocb.aio_buf = reinterpret_cast<uintptr_t>(buffer);
    iocb.aio_nbytes = len;
    return iocb;
}

inline
linux_abi::iocb
make_write_iocb(int fd, uint64_t offset, const void* buffer, size_t len) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::PWRITE;
    iocb.aio_fildes = fd;
    iocb.aio_offset = offset;
    iocb.aio_buf = reinterpret_cast<uintptr_t>(buffer);
    iocb.aio_nbytes = len;
    return iocb;
}

inline
linux_abi::iocb
make_readv_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::PREADV;
    iocb.aio_fildes = fd;
    iocb.aio_offset = offset;
    iocb.aio_buf = reinterpret_cast<uintptr_t>(iov);
    iocb.aio_nbytes = niov;
    return iocb;
}

inline
linux_abi::iocb
make_writev_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::PWRITEV;
    iocb.aio_fildes = fd;
    iocb.aio_offset = offset;
    iocb.aio_buf = reinterpret_cast<uintptr_t>(iov);
    iocb.aio_nbytes = niov;
    return iocb;
}

inline
linux_abi::iocb
make_poll_iocb(int fd, uint32_t events) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::POLL;
    iocb.aio_fildes = fd;
    iocb.aio_buf = events;
    return iocb;
}

inline
linux_abi::iocb
make_fdsync_iocb(int fd) {
    linux_abi::iocb iocb{};
    iocb.aio_lio_opcode = linux_abi::iocb_cmd::FDSYNC;
    iocb.aio_fildes = fd;
    return iocb;
}

inline
void
set_user_data(linux_abi::iocb& iocb, void* data) {
    iocb.aio_data = reinterpret_cast<uintptr_t>(data);
}

template <typename T>
inline T* get_user_data(const linux_abi::iocb& iocb) noexcept {
    return reinterpret_cast<T*>(uintptr_t(iocb.aio_data));
}

template <typename T>
inline T* get_user_data(const linux_abi::io_event& ev) noexcept {
    return reinterpret_cast<T*>(uintptr_t(ev.data));
}

inline
void
set_eventfd_notification(linux_abi::iocb& iocb, int eventfd) {
    iocb.aio_flags |= linux_abi::IOCB_FLAG_RESFD;
    iocb.aio_resfd = eventfd;
}

inline
linux_abi::iocb*
get_iocb(const linux_abi::io_event& ev) {
    return reinterpret_cast<linux_abi::iocb*>(uintptr_t(ev.obj));
}

inline
void
set_nowait(linux_abi::iocb& iocb, bool nowait) {
#ifdef RWF_NOWAIT
    if (aio_nowait_supported) {
        if (nowait) {
            iocb.aio_rw_flags |= RWF_NOWAIT;
        } else {
            iocb.aio_rw_flags &= ~RWF_NOWAIT;
        }
    }
#endif
}

}


}
Commit	Line	Data
11fdf7f2 TL	1	/*
	2	* This file is open source software, licensed to you under the terms
	3	* of the Apache License, Version 2.0 (the "License"). See the NOTICE file
	4	* distributed with this work for additional information regarding copyright
	5	* ownership. You may not use this file except in compliance with the License.
	6	*
	7	* You may obtain a copy of the License at
	8	*
	9	* http://www.apache.org/licenses/LICENSE-2.0
	10	*
	11	* Unless required by applicable law or agreed to in writing,
	12	* software distributed under the License is distributed on an
	13	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	14	* KIND, either express or implied. See the License for the
	15	* specific language governing permissions and limitations
	16	* under the License.
	17	*/
	18	/*
	19	* Copyright (C) 2017 ScyllaDB
	20	*/
	21
	22	#pragma once
	23
	24	#include <endian.h>
	25	#include <sys/time.h>
	26	#include <sys/uio.h>
	27	#include <signal.h>
	28	#include <cstdint>
	29
	30	namespace seastar {
	31
	32	namespace internal {
	33
	34	namespace linux_abi {
	35
	36	using aio_context_t = unsigned long;
	37
	38	enum class iocb_cmd : uint16_t {
	39	PREAD = 0,
	40	PWRITE = 1,
	41	FSYNC = 2,
	42	FDSYNC = 3,
	43	POLL = 5,
	44	NOOP = 6,
	45	PREADV = 7,
	46	PWRITEV = 8,
	47	};
	48
	49	struct io_event {
	50	uint64_t data;
	51	uint64_t obj;
	52	int64_t res;
	53	int64_t res2;
	54	};
	55
	56	constexpr int IOCB_FLAG_RESFD = 1;
	57
	58	struct iocb {
	59	uint64_t aio_data;
	60
	61	#if __BYTE_ORDER == __LITTLE_ENDIAN
	62	uint32_t aio_key;
	63	int32_t aio_rw_flags;
	64	#elif __BYTE_ORDER == __BIG_ENDIAN
65	int32_t aio_rw_flags;
66	uint32_t aio_key;
67	#else
68	#error bad byteorder
69	#endif
70
71	iocb_cmd aio_lio_opcode;
72	int16_t aio_reqprio;
73	uint32_t aio_fildes;
74
75	uint64_t aio_buf;
76	uint64_t aio_nbytes;
77	int64_t aio_offset;
78
79	uint64_t aio_reserved2;
80
81	uint32_t aio_flags;
82
83	uint32_t aio_resfd;
84	};
85
86	struct aio_sigset {
87	const sigset_t *sigmask;
88	size_t sigsetsize;
89	};
90
91	}
92
93	linux_abi::iocb make_read_iocb(int fd, uint64_t offset, void* buffer, size_t len);
94	linux_abi::iocb make_write_iocb(int fd, uint64_t offset, const void* buffer, size_t len);
95	linux_abi::iocb make_readv_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov);
96	linux_abi::iocb make_writev_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov);
97	linux_abi::iocb make_poll_iocb(int fd, uint32_t events);
98
99	void set_user_data(linux_abi::iocb& iocb, void* data);
11fdf7f2 TL	100	void set_nowait(linux_abi::iocb& iocb, bool nowait);
	101
	102	void set_eventfd_notification(linux_abi::iocb& iocb, int eventfd);
	103
	104	linux_abi::iocb* get_iocb(const linux_abi::io_event& ioev);
	105
	106	int io_setup(int nr_events, linux_abi::aio_context_t* io_context);
20effc67	107	int io_destroy(linux_abi::aio_context_t io_context) noexcept;
11fdf7f2 TL	108	int io_submit(linux_abi::aio_context_t io_context, long nr, linux_abi::iocb** iocbs);
	109	int io_cancel(linux_abi::aio_context_t io_context, linux_abi::iocb* iocb, linux_abi::io_event* result);
	110	int io_getevents(linux_abi::aio_context_t io_context, long min_nr, long nr, linux_abi::io_event* events, const ::timespec* timeout,
	111	bool force_syscall = false);
	112	int io_pgetevents(linux_abi::aio_context_t io_context, long min_nr, long nr, linux_abi::io_event* events, const ::timespec* timeout, const sigset_t* sigmask,
	113	bool force_syscall = false);
	114
9f95a23c TL	115	void setup_aio_context(size_t nr, linux_abi::aio_context_t* io_context);
9f95a23c TL	116
11fdf7f2 TL	117	}
11fdf7f2 TL	118
f67539c2 TL	119	extern bool aio_nowait_supported;
f67539c2 TL	120
11fdf7f2 TL	121	namespace internal {
	122
	123	inline
	124	linux_abi::iocb
	125	make_read_iocb(int fd, uint64_t offset, void* buffer, size_t len) {
	126	linux_abi::iocb iocb{};
	127	iocb.aio_lio_opcode = linux_abi::iocb_cmd::PREAD;
	128	iocb.aio_fildes = fd;
	129	iocb.aio_offset = offset;
	130	iocb.aio_buf = reinterpret_cast<uintptr_t>(buffer);
	131	iocb.aio_nbytes = len;
	132	return iocb;
	133	}
	134
	135	inline
	136	linux_abi::iocb
	137	make_write_iocb(int fd, uint64_t offset, const void* buffer, size_t len) {
	138	linux_abi::iocb iocb{};
	139	iocb.aio_lio_opcode = linux_abi::iocb_cmd::PWRITE;
	140	iocb.aio_fildes = fd;
	141	iocb.aio_offset = offset;
	142	iocb.aio_buf = reinterpret_cast<uintptr_t>(buffer);
	143	iocb.aio_nbytes = len;
	144	return iocb;
	145	}
	146
	147	inline
	148	linux_abi::iocb
	149	make_readv_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov) {
	150	linux_abi::iocb iocb{};
	151	iocb.aio_lio_opcode = linux_abi::iocb_cmd::PREADV;
	152	iocb.aio_fildes = fd;
	153	iocb.aio_offset = offset;
	154	iocb.aio_buf = reinterpret_cast<uintptr_t>(iov);
	155	iocb.aio_nbytes = niov;
	156	return iocb;
	157	}
	158
	159	inline
	160	linux_abi::iocb
	161	make_writev_iocb(int fd, uint64_t offset, const ::iovec* iov, size_t niov) {
	162	linux_abi::iocb iocb{};
	163	iocb.aio_lio_opcode = linux_abi::iocb_cmd::PWRITEV;
	164	iocb.aio_fildes = fd;
	165	iocb.aio_offset = offset;
	166	iocb.aio_buf = reinterpret_cast<uintptr_t>(iov);
	167	iocb.aio_nbytes = niov;
	168	return iocb;
	169	}
	170
	171	inline
	172	linux_abi::iocb
	173	make_poll_iocb(int fd, uint32_t events) {
	174	linux_abi::iocb iocb{};
	175	iocb.aio_lio_opcode = linux_abi::iocb_cmd::POLL;
	176	iocb.aio_fildes = fd;
	177	iocb.aio_buf = events;
	178	return iocb;
	179	}
	180
9f95a23c TL	181	inline
	182	linux_abi::iocb
	183	make_fdsync_iocb(int fd) {
	184	linux_abi::iocb iocb{};
	185	iocb.aio_lio_opcode = linux_abi::iocb_cmd::FDSYNC;
	186	iocb.aio_fildes = fd;
	187	return iocb;
	188	}
	189
11fdf7f2 TL	190	inline
	191	void
	192	set_user_data(linux_abi::iocb& iocb, void* data) {
	193	iocb.aio_data = reinterpret_cast<uintptr_t>(data);
	194	}
	195
1e59de90 TL	196	template <typename T>
	197	inline T* get_user_data(const linux_abi::iocb& iocb) noexcept {
	198	return reinterpret_cast<T*>(uintptr_t(iocb.aio_data));
	199	}
	200
	201	template <typename T>
	202	inline T* get_user_data(const linux_abi::io_event& ev) noexcept {
	203	return reinterpret_cast<T*>(uintptr_t(ev.data));
11fdf7f2 TL	204	}
	205
	206	inline
	207	void
	208	set_eventfd_notification(linux_abi::iocb& iocb, int eventfd) {
	209	iocb.aio_flags \|= linux_abi::IOCB_FLAG_RESFD;
	210	iocb.aio_resfd = eventfd;
	211	}
	212
	213	inline
	214	linux_abi::iocb*
	215	get_iocb(const linux_abi::io_event& ev) {
	216	return reinterpret_cast<linux_abi::iocb*>(uintptr_t(ev.obj));
	217	}
	218
	219	inline
	220	void
	221	set_nowait(linux_abi::iocb& iocb, bool nowait) {
	222	#ifdef RWF_NOWAIT
f67539c2 TL	223	if (aio_nowait_supported) {
	224	if (nowait) {
	225	iocb.aio_rw_flags \|= RWF_NOWAIT;
	226	} else {
	227	iocb.aio_rw_flags &= ~RWF_NOWAIT;
	228	}
11fdf7f2 TL	229	}
	230	#endif
	231	}
	232
	233	}
	234
	235
	236	}
	237