mirrors/fuseuring
2
0
Fork 0
mirror of https://github.com/uroni/fuseuring.git synced 2025-10-26 11:19:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Martin Raiber 2020-11-07 17:24:12 +01:00
commit 9508a99297
15 changed files with 2774 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
AUTHORS Normal file
View File

0
ChangeLog Normal file
View File

9
Makefile.am Executable file
View File

@ -0,0 +1,9 @@
ACLOCAL_AMFLAGS = -I m4
bin_PROGRAMS = fuseuring
fuseuring_SOURCES = fuse_io_service.cpp main.cpp fuseuring_main.cpp
fuseuring_LDADD = $(PTHREAD_LIBS) -luring
fuseuring_CXXFLAGS = $(PTHREAD_CFLAGS) -std=c++2a -D_FILE_OFFSET_BITS=64
noinst_HEADERS = fuse_io_service.h fuseuring_main.h fuse_kernel.h

0
NEWS Normal file
View File

1
README Normal file
View File

@ -0,0 +1 @@
Please see readme.md.

23
bench.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -ex
FMNT=/media/test
BMNT=/media/bench
mkdir -p "$FMNT"
./fuseuring /tmp/backing_file.img "$FMNT" $((500*1024*1024)) 5000 &
while ! test -e "$FMNT/volume"; do sleep 1; done
LODEV=$(losetup --find --show "$FMNT/volume" --direct-io=on)
mkfs.ext4 -F $LODEV
mount $LODEV "$BMNT"
losetup -d $LODEV
cp /usr/share/doc/fio/examples/ssd-test.fio ./
sed -i 's/iodepth=4/iodepth=1024/g' ssd-test.fio
sed -i 's/size=10g/size=400m/g' ssd-test.fio
sed -i "s@directory=/mount-point-of-ssd@directory=$BMNT@g" ssd-test.fio
fio ssd-test.fio
umount "$BMNT"
umount "$FMNT"

69
config.h Normal file
View File

@ -0,0 +1,69 @@
/* config.h. Generated from config.h.in by configure. */
/* config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define if you have POSIX threads libraries and header files. */
#define HAVE_PTHREAD 1
/* Define to 1 if you have the <pthread.h> header file. */
#define HAVE_PTHREAD_H 1
/* pthread has GNU extension thread_setname_np */
#define HAVE_PTHREAD_SETNAME_NP 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Name of package */
#define PACKAGE "fuseuring"
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "martin@urbackup.org"
/* Define to the full name of this package. */
#define PACKAGE_NAME "fuseuring"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "fuseuring 0.1.0.0"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "fuseuring"
/* Define to the home page for this package. */
#define PACKAGE_URL ""
/* Define to the version of this package. */
#define PACKAGE_VERSION "0.1.0.0"
/* Define to necessary symbol if this constant uses a non-standard name on
your system. */
/* #undef PTHREAD_CREATE_JOINABLE */
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Version number of package */
#define VERSION "0.1.0.0"

39
configure.ac Executable file
View File

@ -0,0 +1,39 @@
# -*- Autoconf -*-
# Process this file with autoconf to produce a configure script.
AC_PREREQ(2.61)
AC_INIT([fuseuring], [0.1.0.0], [martin@urbackup.org])
AC_CONFIG_SRCDIR([main.cpp])
AC_CONFIG_HEADER([config.h])
AC_CONFIG_MACRO_DIR([m4])
AC_CANONICAL_SYSTEM
AM_INIT_AUTOMAKE([subdir-objects tar-ustar])
# Checks for programs.
AC_PROG_CXX
AC_PROG_CC
AM_PROG_CC_C_O
AX_PTHREAD
if !($HAVE_PTHREAD)
then
echo "Sorry, your system needs the pthread library."
echo "Either install it or give up."
exit 1
fi
AC_LANG([C++])
# Checks for libraries.
# Checks for header files.
AC_CHECK_HEADERS([pthread.h])
# Checks for library functions.
AC_SEARCH_LIBS([clock_gettime], [rt posix4],
[test "$ac_cv_search_clock_gettime" = "none required" || LIBS="$LIBS $ac_cv_search_clock_gettime"],
[AC_MSG_FAILURE([No library for clock_gettime found])] )
AC_CONFIG_FILES([Makefile])
AC_OUTPUT

174
fuse_io_service.cpp Executable file
View File

@ -0,0 +1,174 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// Copyright (C) Martin Raiber
#include "fuse_io_service.h"
#include "../Interface/Server.h"
#include <liburing.h>
#include <iostream>
fuse_io_service::fuse_io_service(FuseRing fuse_ring)
: fuse_ring(fuse_ring)
{
}
int fuse_io_service::fuseuring_handle_cqe(struct io_uring_cqe *cqe)
{
if(cqe->user_data==0)
{
DBG_PRINT(std::cerr << "Cqe no user_data" << std::endl);
return 0;
}
IoUringAwaiter<int>::IoUringAwaiterRes* res = reinterpret_cast<IoUringAwaiter<int>::IoUringAwaiterRes*>(cqe->user_data);
res->res = cqe->res;
DBG_PRINT(std::cout << "Cqe res "<< cqe->res << std::endl);
--res->gres->tocomplete;
if(res->gres->tocomplete==0)
{
DBG_PRINT(std::cerr << "Resume cqe..." << std::endl);
res->gres->awaiter.resume();
}
return 0;
}
int fuse_io_service::fuseuring_submit(bool block)
{
if(fuse_ring.ring_submit)
{
int rc;
if(block)
rc = io_uring_submit_and_wait(fuse_ring.ring, 1);
else
rc = io_uring_submit(fuse_ring.ring);
if(rc<0)
{
perror("Error submitting to fuse io_uring.");
return 18;
}
fuse_ring.ring_submit=false;
}
return 0;
}
int fuse_io_service::run_sqe_awaiters()
{
if(sqe_awaiters!=nullptr)
{
if(int rc; (rc=fuseuring_submit(false))!=0)
return rc;
}
while(sqe_awaiters!=nullptr)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(fuse_ring.ring);
if(sqe)
{
IoUringSqeAwaiter* curr = sqe_awaiters;
sqe_awaiters = curr->next;
curr->sqe = sqe;
curr->awaiter.resume();
}
else
{
break;
}
}
return 0;
}
int fuse_io_service::run(queue_fuse_read_t queue_read)
{
fuse_ring.ring_submit = false;
std::vector<io_uring_task<int> > tasks;
while(true)
{
if(int rc; (rc=run_sqe_awaiters())!=0)
return rc;
while(!fuse_ring.ios.empty())
{
io_uring_task task = queue_read(*this);
if(task.has_res())
{
int rc = task.res();
if(rc<0)
{
std::cerr << "Error after running task (1): " << rc << std::endl;
return 20;
}
}
else
{
tasks.push_back(std::move(task));
}
}
if(int rc; (rc=run_sqe_awaiters())!=0)
return rc;
if(int rc; (rc=fuseuring_submit(true))!=0)
return rc;
int nr_comp = 0;
while(true)
{
struct io_uring_cqe *cqe;
if(nr_comp==0)
{
int rc = io_uring_wait_cqe(fuse_ring.ring, &cqe);
if(rc<0)
{
perror("Waiting for fuse iouring cqe failed.");
return 16;
}
}
else
{
int rc = io_uring_peek_cqe(fuse_ring.ring, &cqe);
if(rc!=0)
{
break;
}
}
int rc = fuseuring_handle_cqe(cqe);
if(rc<0)
{
std::cerr << "Error handling cqe rc=" << rc << std::endl;
return 17;
}
io_uring_cqe_seen(fuse_ring.ring, cqe);
++nr_comp;
}
std::vector<io_uring_task<int> > new_tasks;
for(auto& task: tasks)
{
if(task.has_res())
{
int rc = task.res();
if(rc<0)
{
std::cerr << "Error running task (2). rc: " << rc << std::endl;
return 18;
}
}
else
{
new_tasks.push_back(std::move(task));
}
}
std::swap(tasks, new_tasks);
}
}

373
fuse_io_service.h Executable file
View File

@ -0,0 +1,373 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// Copyright (C) Martin Raiber
#pragma once
#include <coroutine>
#include <vector>
#include <liburing.h>
#include <stack>
#include <utility>
#include <assert.h>
#include <iostream>
#define DBG_PRINT(x)
/*
for clang and libc++
namespace std
{
template<typename T = void>
using coroutine_handle = std::experimental::coroutine_handle<T>;
using suspend_never = std::experimental::suspend_never;
using suspend_always = std::experimental::suspend_always;
}*/
static uint64_t handle_v(std::coroutine_handle<> p_awaiter)
{
return (uint64_t)*((uint64_t*)&p_awaiter);
}
struct fuse_io_service
{
template<typename T>
struct IoUringAwaiter
{
struct IoUringAwaiterGlobalRes
{
size_t tocomplete;
std::coroutine_handle<> awaiter;
};
struct IoUringAwaiterRes
{
IoUringAwaiterRes() noexcept
: res(-1) {}
int res;
IoUringAwaiterGlobalRes* gres;
};
IoUringAwaiter(std::vector<io_uring_sqe*> sqes) noexcept
{
awaiter_res.resize(sqes.size());
global_res.tocomplete = sqes.size();
for(size_t i=0;i<sqes.size();++i)
{
awaiter_res[i].gres = &global_res;
sqes[i]->user_data = reinterpret_cast<uint64_t>(&awaiter_res[i]);
}
}
IoUringAwaiter(IoUringAwaiter const&) = delete;
IoUringAwaiter(IoUringAwaiter&& other) = delete;
IoUringAwaiter& operator=(IoUringAwaiter&&) = delete;
IoUringAwaiter& operator=(IoUringAwaiter const&) = delete;
bool await_ready() const noexcept
{
return false;
}
void await_suspend(std::coroutine_handle<> p_awaiter) noexcept
{
DBG_PRINT(std::cout << "Await suspend io "<< handle_v(p_awaiter) << std::endl);
global_res.awaiter = p_awaiter;
}
template<typename U = T, std::enable_if_t<std::is_same<U, std::vector<io_uring_sqe*> >::value, int> = 0>
std::vector<int> await_resume() const noexcept
{
std::vector<int> res;
for(auto& sr: awaiter_res)
{
res.push_back(sr.res);
}
return res;
}
template<typename U = T, std::enable_if_t<std::is_same<U, io_uring_sqe*>::value, int> = 0>
int await_resume() const noexcept
{
return awaiter_res[0].res;
}
template<typename U = T, std::enable_if_t<std::is_same<U, std::pair<io_uring_sqe*, io_uring_sqe*> >::value, int> = 0>
std::pair<int, int> await_resume() const noexcept
{
return std::make_pair(awaiter_res[0].res, awaiter_res[1].res);
}
private:
IoUringAwaiterGlobalRes global_res;
std::vector<IoUringAwaiterRes> awaiter_res;
};
[[nodiscard]] auto complete(std::vector<io_uring_sqe*> sqes)
{
return IoUringAwaiter<std::vector<io_uring_sqe*> >(sqes);
}
[[nodiscard]] auto complete(io_uring_sqe* sqe)
{
return IoUringAwaiter<io_uring_sqe*>({sqe});
}
[[nodiscard]] auto complete(std::pair<io_uring_sqe*, io_uring_sqe*> sqes)
{
return IoUringAwaiter<std::pair<io_uring_sqe*, io_uring_sqe*> >({sqes.first, sqes.second});
}
struct IoUringSqeAwaiter;
IoUringSqeAwaiter* sqe_awaiters = nullptr;
struct IoUringSqeAwaiter
{
IoUringSqeAwaiter(fuse_io_service& io, struct io_uring* uring)
: io(io), next(nullptr)
{
sqe = io_uring_get_sqe(uring);
}
IoUringSqeAwaiter(IoUringSqeAwaiter const&) = delete;
IoUringSqeAwaiter(IoUringSqeAwaiter&& other) = delete;
IoUringSqeAwaiter& operator=(IoUringSqeAwaiter&&) = delete;
IoUringSqeAwaiter& operator=(IoUringSqeAwaiter const&) = delete;
bool await_ready() const noexcept
{
return sqe!=nullptr;
}
void await_suspend(std::coroutine_handle<> p_awaiter) noexcept
{
DBG_PRINT(std::cout << "Await suspend sqe "<< handle_v(p_awaiter) << std::endl);
awaiter = p_awaiter;
if(io.sqe_awaiters!=nullptr)
{
next = io.sqe_awaiters;
}
io.sqe_awaiters = this;
}
struct io_uring_sqe* await_resume() const noexcept
{
return sqe;
}
fuse_io_service& io;
std::coroutine_handle<> awaiter;
IoUringSqeAwaiter* next;
struct io_uring_sqe *sqe;
};
[[nodiscard]] auto get_sqe()
{
fuse_ring.ring_submit=true;
return IoUringSqeAwaiter(*this, fuse_ring.ring);
}
template<typename T>
struct io_uring_promise_type
{
using promise_type = io_uring_promise_type<T>;
using handle = std::coroutine_handle<promise_type>;
enum class e_res_state
{
Init,
Detached,
Res
};
io_uring_promise_type()
: res_state(e_res_state::Init) {}
auto get_return_object()
{
return io_uring_task<T>{handle::from_promise(*this)};
}
void return_value(T v)
{
res_state = e_res_state::Res;
res = v;
}
auto initial_suspend() {
return std::suspend_never{};
}
auto final_suspend() noexcept {
struct final_awaiter : std::suspend_always
{
final_awaiter(promise_type* promise)
:promise(promise) {}
void await_suspend(std::coroutine_handle<> p_awaiter) const noexcept
{
if(promise->res_state==e_res_state::Detached)
{
DBG_PRINT(std::cout << "promise final detached" << std::endl);
if(promise->awaiter)
promise->awaiter.destroy();
handle::from_promise(*promise).destroy();
}
else if(promise->awaiter)
{
DBG_PRINT(std::cout << "promise final await resume" << std::endl);
promise->awaiter.resume();
}
else
{
DBG_PRINT(std::cout << "promise final no awaiter" << std::endl);
}
}
private:
promise_type* promise;
};
return final_awaiter(this);
}
void unhandled_exception()
{
abort();
}
std::coroutine_handle<> awaiter;
e_res_state res_state;
T res;
};
template<typename T>
struct [[nodiscard]] io_uring_task
{
using promise_type = io_uring_promise_type<T>;
using handle = std::coroutine_handle<promise_type>;
io_uring_task(io_uring_task const&) = delete;
io_uring_task(io_uring_task&& other) noexcept
: coro_h(std::exchange(other.coro_h, {}))
{
}
io_uring_task& operator=(io_uring_task&&) = delete;
io_uring_task& operator=(io_uring_task const&) = delete;
io_uring_task(handle h) noexcept
: coro_h(h)
{
}
~io_uring_task() noexcept
{
if(coro_h)
{
if(!coro_h.done())
{
DBG_PRINT(std::cout << "Detach" << std::endl);
coro_h.promise().res_state = promise_type::e_res_state::Detached;
}
else
{
DBG_PRINT(std::cout << "Destroy" << std::endl);
coro_h.destroy();
}
}
}
bool has_res() const noexcept
{
return coro_h.promise().res_state == promise_type::e_res_state::Res;
}
T res() const noexcept
{
assert(has_res());
return coro_h.promise().res;
}
bool await_ready() const noexcept
{
bool r = has_res();
if(r)
{
DBG_PRINT(std::cout << "Await is ready" << std::endl);
}
else
{
DBG_PRINT(std::cout << "Await is not ready" << std::endl);
}
return r;
}
template<typename U>
void await_suspend(std::coroutine_handle<io_uring_promise_type<U> > p_awaiter) noexcept
{
DBG_PRINT(std::cout << "Await task " << (uint64_t)this << " suspend "<< handle_v(p_awaiter) << " prev " << handle_v(coro_h.promise().awaiter) << std::endl);
coro_h.promise().awaiter = p_awaiter;
}
T await_resume() const noexcept
{
return res();
}
private:
handle coro_h;
};
struct SFuseIo
{
char type;
int pipe[2];
char* header_buf;
size_t header_buf_idx;
char* scratch_buf;
size_t scratch_buf_idx;
};
struct FuseRing
{
std::stack<SFuseIo*> ios;
struct io_uring* ring;
int fd;
bool ring_submit;
size_t max_bufsize;
int backing_fd;
int backing_fd_orig;
uint64_t backing_f_size;
};
FuseRing fuse_ring;
fuse_io_service(FuseRing fuse_ring);
typedef fuse_io_service::io_uring_task<int> (*queue_fuse_read_t)(fuse_io_service& io);
int run(queue_fuse_read_t queue_read);
SFuseIo* get_fuse_io()
{
fuse_io_service::SFuseIo* fuse_io = fuse_ring.ios.top();
fuse_ring.ios.pop();
return fuse_io;
}
void release_fuse_io(SFuseIo* fuse_io)
{
fuse_ring.ios.push(fuse_io);
}
private:
int fuseuring_handle_cqe(struct io_uring_cqe *cqe);
int fuseuring_submit(bool block);
int run_sqe_awaiters();
};

848
fuse_kernel.h Executable file
View File

@ -0,0 +1,848 @@
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
/*
This file defines the kernel interface of FUSE
Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
This -- and only this -- header file may also be distributed under
the terms of the BSD Licence as follows:
Copyright (C) 2001-2007 Miklos Szeredi. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
*/
/*
* This file defines the kernel interface of FUSE
*
* Protocol changelog:
*
* 7.9:
* - new fuse_getattr_in input argument of GETATTR
* - add lk_flags in fuse_lk_in
* - add lock_owner field to fuse_setattr_in, fuse_read_in and fuse_write_in
* - add blksize field to fuse_attr
* - add file flags field to fuse_read_in and fuse_write_in
* - Add ATIME_NOW and MTIME_NOW flags to fuse_setattr_in
*
* 7.10
* - add nonseekable open flag
*
* 7.11
* - add IOCTL message
* - add unsolicited notification support
* - add POLL message and NOTIFY_POLL notification
*
* 7.12
* - add umask flag to input argument of create, mknod and mkdir
* - add notification messages for invalidation of inodes and
* directory entries
*
* 7.13
* - make max number of background requests and congestion threshold
* tunables
*
* 7.14
* - add splice support to fuse device
*
* 7.15
* - add store notify
* - add retrieve notify
*
* 7.16
* - add BATCH_FORGET request
* - FUSE_IOCTL_UNRESTRICTED shall now return with array of 'struct
* fuse_ioctl_iovec' instead of ambiguous 'struct iovec'
* - add FUSE_IOCTL_32BIT flag
*
* 7.17
* - add FUSE_FLOCK_LOCKS and FUSE_RELEASE_FLOCK_UNLOCK
*
* 7.18
* - add FUSE_IOCTL_DIR flag
* - add FUSE_NOTIFY_DELETE
*
* 7.19
* - add FUSE_FALLOCATE
*
* 7.20
* - add FUSE_AUTO_INVAL_DATA
*
* 7.21
* - add FUSE_READDIRPLUS
* - send the requested events in POLL request
*
* 7.22
* - add FUSE_ASYNC_DIO
*
* 7.23
* - add FUSE_WRITEBACK_CACHE
* - add time_gran to fuse_init_out
* - add reserved space to fuse_init_out
* - add FATTR_CTIME
* - add ctime and ctimensec to fuse_setattr_in
* - add FUSE_RENAME2 request
* - add FUSE_NO_OPEN_SUPPORT flag
*
* 7.24
* - add FUSE_LSEEK for SEEK_HOLE and SEEK_DATA support
*
* 7.25
* - add FUSE_PARALLEL_DIROPS
*
* 7.26
* - add FUSE_HANDLE_KILLPRIV
* - add FUSE_POSIX_ACL
*
* 7.27
* - add FUSE_ABORT_ERROR
*
* 7.28
* - add FUSE_COPY_FILE_RANGE
* - add FOPEN_CACHE_DIR
* - add FUSE_MAX_PAGES, add max_pages to init_out
* - add FUSE_CACHE_SYMLINKS
*
* 7.29
* - add FUSE_NO_OPENDIR_SUPPORT flag
*
* 7.30
* - add FUSE_EXPLICIT_INVAL_DATA
* - add FUSE_IOCTL_COMPAT_X32
*
* 7.31
* - add FUSE_WRITE_KILL_PRIV flag
*/
#ifndef _LINUX_FUSE_H
#define _LINUX_FUSE_H
#ifdef __KERNEL__
#include <linux/types.h>
#else
#include <stdint.h>
#endif
/*
* Version negotiation:
*
* Both the kernel and userspace send the version they support in the
* INIT request and reply respectively.
*
* If the major versions match then both shall use the smallest
* of the two minor versions for communication.
*
* If the kernel supports a larger major version, then userspace shall
* reply with the major version it supports, ignore the rest of the
* INIT message and expect a new INIT message from the kernel with a
* matching major version.
*
* If the library supports a larger major version, then it shall fall
* back to the major protocol version sent by the kernel for
* communication and reply with that major version (and an arbitrary
* supported minor version).
*/
/** Version number of this interface */
#define FUSE_KERNEL_VERSION 7
/** Minor version number of this interface */
#define FUSE_KERNEL_MINOR_VERSION 31
/** The node ID of the root inode */
#define FUSE_ROOT_ID 1
/* Make sure all structures are padded to 64bit boundary, so 32bit
userspace works under 64bit kernels */
struct fuse_attr {
uint64_t ino;
uint64_t size;
uint64_t blocks;
uint64_t atime;
uint64_t mtime;
uint64_t ctime;
uint32_t atimensec;
uint32_t mtimensec;
uint32_t ctimensec;
uint32_t mode;
uint32_t nlink;
uint32_t uid;
uint32_t gid;
uint32_t rdev;
uint32_t blksize;
uint32_t padding;
};
struct fuse_kstatfs {
uint64_t blocks;
uint64_t bfree;
uint64_t bavail;
uint64_t files;
uint64_t ffree;
uint32_t bsize;
uint32_t namelen;
uint32_t frsize;
uint32_t padding;
uint32_t spare[6];
};
struct fuse_file_lock {
uint64_t start;
uint64_t end;
uint32_t type;
uint32_t pid; /* tgid */
};
/**
* Bitmasks for fuse_setattr_in.valid
*/
#define FATTR_MODE (1 << 0)
#define FATTR_UID (1 << 1)
#define FATTR_GID (1 << 2)
#define FATTR_SIZE (1 << 3)
#define FATTR_ATIME (1 << 4)
#define FATTR_MTIME (1 << 5)
#define FATTR_FH (1 << 6)
#define FATTR_ATIME_NOW (1 << 7)
#define FATTR_MTIME_NOW (1 << 8)
#define FATTR_LOCKOWNER (1 << 9)
#define FATTR_CTIME (1 << 10)
/**
* Flags returned by the OPEN request
*
* FOPEN_DIRECT_IO: bypass page cache for this open file
* FOPEN_KEEP_CACHE: don't invalidate the data cache on open
* FOPEN_NONSEEKABLE: the file is not seekable
* FOPEN_CACHE_DIR: allow caching this directory
* FOPEN_STREAM: the file is stream-like (no file position at all)
*/
#define FOPEN_DIRECT_IO (1 << 0)
#define FOPEN_KEEP_CACHE (1 << 1)
#define FOPEN_NONSEEKABLE (1 << 2)
#define FOPEN_CACHE_DIR (1 << 3)
#define FOPEN_STREAM (1 << 4)
/**
* INIT request/reply flags
*
* FUSE_ASYNC_READ: asynchronous read requests
* FUSE_POSIX_LOCKS: remote locking for POSIX file locks
* FUSE_FILE_OPS: kernel sends file handle for fstat, etc... (not yet supported)
* FUSE_ATOMIC_O_TRUNC: handles the O_TRUNC open flag in the filesystem
* FUSE_EXPORT_SUPPORT: filesystem handles lookups of "." and ".."
* FUSE_BIG_WRITES: filesystem can handle write size larger than 4kB
* FUSE_DONT_MASK: don't apply umask to file mode on create operations
* FUSE_SPLICE_WRITE: kernel supports splice write on the device
* FUSE_SPLICE_MOVE: kernel supports splice move on the device
* FUSE_SPLICE_READ: kernel supports splice read on the device
* FUSE_FLOCK_LOCKS: remote locking for BSD style file locks
* FUSE_HAS_IOCTL_DIR: kernel supports ioctl on directories
* FUSE_AUTO_INVAL_DATA: automatically invalidate cached pages
* FUSE_DO_READDIRPLUS: do READDIRPLUS (READDIR+LOOKUP in one)
* FUSE_READDIRPLUS_AUTO: adaptive readdirplus
* FUSE_ASYNC_DIO: asynchronous direct I/O submission
* FUSE_WRITEBACK_CACHE: use writeback cache for buffered writes
* FUSE_NO_OPEN_SUPPORT: kernel supports zero-message opens
* FUSE_PARALLEL_DIROPS: allow parallel lookups and readdir
* FUSE_HANDLE_KILLPRIV: fs handles killing suid/sgid/cap on write/chown/trunc
* FUSE_POSIX_ACL: filesystem supports posix acls
* FUSE_ABORT_ERROR: reading the device after abort returns ECONNABORTED
* FUSE_MAX_PAGES: init_out.max_pages contains the max number of req pages
* FUSE_CACHE_SYMLINKS: cache READLINK responses
* FUSE_NO_OPENDIR_SUPPORT: kernel supports zero-message opendir
* FUSE_EXPLICIT_INVAL_DATA: only invalidate cached pages on explicit request
*/
#define FUSE_ASYNC_READ (1 << 0)
#define FUSE_POSIX_LOCKS (1 << 1)
#define FUSE_FILE_OPS (1 << 2)
#define FUSE_ATOMIC_O_TRUNC (1 << 3)
#define FUSE_EXPORT_SUPPORT (1 << 4)
#define FUSE_BIG_WRITES (1 << 5)
#define FUSE_DONT_MASK (1 << 6)
#define FUSE_SPLICE_WRITE (1 << 7)
#define FUSE_SPLICE_MOVE (1 << 8)
#define FUSE_SPLICE_READ (1 << 9)
#define FUSE_FLOCK_LOCKS (1 << 10)
#define FUSE_HAS_IOCTL_DIR (1 << 11)
#define FUSE_AUTO_INVAL_DATA (1 << 12)
#define FUSE_DO_READDIRPLUS (1 << 13)
#define FUSE_READDIRPLUS_AUTO (1 << 14)
#define FUSE_ASYNC_DIO (1 << 15)
#define FUSE_WRITEBACK_CACHE (1 << 16)
#define FUSE_NO_OPEN_SUPPORT (1 << 17)
#define FUSE_PARALLEL_DIROPS (1 << 18)
#define FUSE_HANDLE_KILLPRIV (1 << 19)
#define FUSE_POSIX_ACL (1 << 20)
#define FUSE_ABORT_ERROR (1 << 21)
#define FUSE_MAX_PAGES (1 << 22)
#define FUSE_CACHE_SYMLINKS (1 << 23)
#define FUSE_NO_OPENDIR_SUPPORT (1 << 24)
#define FUSE_EXPLICIT_INVAL_DATA (1 << 25)
/**
* CUSE INIT request/reply flags
*
* CUSE_UNRESTRICTED_IOCTL: use unrestricted ioctl
*/
#define CUSE_UNRESTRICTED_IOCTL (1 << 0)
/**
* Release flags
*/
#define FUSE_RELEASE_FLUSH (1 << 0)
#define FUSE_RELEASE_FLOCK_UNLOCK (1 << 1)
/**
* Getattr flags
*/
#define FUSE_GETATTR_FH (1 << 0)
/**
* Lock flags
*/
#define FUSE_LK_FLOCK (1 << 0)
/**
* WRITE flags
*
* FUSE_WRITE_CACHE: delayed write from page cache, file handle is guessed
* FUSE_WRITE_LOCKOWNER: lock_owner field is valid
* FUSE_WRITE_KILL_PRIV: kill suid and sgid bits
*/
#define FUSE_WRITE_CACHE (1 << 0)
#define FUSE_WRITE_LOCKOWNER (1 << 1)
#define FUSE_WRITE_KILL_PRIV (1 << 2)
/**
* Read flags
*/
#define FUSE_READ_LOCKOWNER (1 << 1)
/**
* Ioctl flags
*
* FUSE_IOCTL_COMPAT: 32bit compat ioctl on 64bit machine
* FUSE_IOCTL_UNRESTRICTED: not restricted to well-formed ioctls, retry allowed
* FUSE_IOCTL_RETRY: retry with new iovecs
* FUSE_IOCTL_32BIT: 32bit ioctl
* FUSE_IOCTL_DIR: is a directory
* FUSE_IOCTL_COMPAT_X32: x32 compat ioctl on 64bit machine (64bit time_t)
*
* FUSE_IOCTL_MAX_IOV: maximum of in_iovecs + out_iovecs
*/
#define FUSE_IOCTL_COMPAT (1 << 0)
#define FUSE_IOCTL_UNRESTRICTED (1 << 1)
#define FUSE_IOCTL_RETRY (1 << 2)
#define FUSE_IOCTL_32BIT (1 << 3)
#define FUSE_IOCTL_DIR (1 << 4)
#define FUSE_IOCTL_COMPAT_X32 (1 << 5)
#define FUSE_IOCTL_MAX_IOV 256
/**
* Poll flags
*
* FUSE_POLL_SCHEDULE_NOTIFY: request poll notify
*/
#define FUSE_POLL_SCHEDULE_NOTIFY (1 << 0)
/**
* Fsync flags
*
* FUSE_FSYNC_FDATASYNC: Sync data only, not metadata
*/
#define FUSE_FSYNC_FDATASYNC (1 << 0)
enum fuse_opcode {
FUSE_LOOKUP = 1,
FUSE_FORGET = 2, /* no reply */
FUSE_GETATTR = 3,
FUSE_SETATTR = 4,
FUSE_READLINK = 5,
FUSE_SYMLINK = 6,
FUSE_MKNOD = 8,
FUSE_MKDIR = 9,
FUSE_UNLINK = 10,
FUSE_RMDIR = 11,
FUSE_RENAME = 12,
FUSE_LINK = 13,
FUSE_OPEN = 14,
FUSE_READ = 15,
FUSE_WRITE = 16,
FUSE_STATFS = 17,
FUSE_RELEASE = 18,
FUSE_FSYNC = 20,
FUSE_SETXATTR = 21,
FUSE_GETXATTR = 22,
FUSE_LISTXATTR = 23,
FUSE_REMOVEXATTR = 24,
FUSE_FLUSH = 25,
FUSE_INIT = 26,
FUSE_OPENDIR = 27,
FUSE_READDIR = 28,
FUSE_RELEASEDIR = 29,
FUSE_FSYNCDIR = 30,
FUSE_GETLK = 31,
FUSE_SETLK = 32,
FUSE_SETLKW = 33,
FUSE_ACCESS = 34,
FUSE_CREATE = 35,
FUSE_INTERRUPT = 36,
FUSE_BMAP = 37,
FUSE_DESTROY = 38,
FUSE_IOCTL = 39,
FUSE_POLL = 40,
FUSE_NOTIFY_REPLY = 41,
FUSE_BATCH_FORGET = 42,
FUSE_FALLOCATE = 43,
FUSE_READDIRPLUS = 44,
FUSE_RENAME2 = 45,
FUSE_LSEEK = 46,
FUSE_COPY_FILE_RANGE = 47,
/* CUSE specific operations */
CUSE_INIT = 4096
};
enum fuse_notify_code {
FUSE_NOTIFY_POLL = 1,
FUSE_NOTIFY_INVAL_INODE = 2,
FUSE_NOTIFY_INVAL_ENTRY = 3,
FUSE_NOTIFY_STORE = 4,
FUSE_NOTIFY_RETRIEVE = 5,
FUSE_NOTIFY_DELETE = 6,
FUSE_NOTIFY_CODE_MAX
};
/* The read buffer is required to be at least 8k, but may be much larger */
#define FUSE_MIN_READ_BUFFER 8192
#define FUSE_COMPAT_ENTRY_OUT_SIZE 120
struct fuse_entry_out {
uint64_t nodeid; /* Inode ID */
uint64_t generation; /* Inode generation: nodeid:gen must
be unique for the fs's lifetime */
uint64_t entry_valid; /* Cache timeout for the name */
uint64_t attr_valid; /* Cache timeout for the attributes */
uint32_t entry_valid_nsec;
uint32_t attr_valid_nsec;
struct fuse_attr attr;
};
struct fuse_forget_in {
uint64_t nlookup;
};
struct fuse_forget_one {
uint64_t nodeid;
uint64_t nlookup;
};
struct fuse_batch_forget_in {
uint32_t count;
uint32_t dummy;
};
struct fuse_getattr_in {
uint32_t getattr_flags;
uint32_t dummy;
uint64_t fh;
};
#define FUSE_COMPAT_ATTR_OUT_SIZE 96
struct fuse_attr_out {
uint64_t attr_valid; /* Cache timeout for the attributes */
uint32_t attr_valid_nsec;
uint32_t dummy;
struct fuse_attr attr;
};
#define FUSE_COMPAT_MKNOD_IN_SIZE 8
struct fuse_mknod_in {
uint32_t mode;
uint32_t rdev;
uint32_t umask;
uint32_t padding;
};
struct fuse_mkdir_in {
uint32_t mode;
uint32_t umask;
};
struct fuse_rename_in {
uint64_t newdir;
};
struct fuse_rename2_in {
uint64_t newdir;
uint32_t flags;
uint32_t padding;
};
struct fuse_link_in {
uint64_t oldnodeid;
};
struct fuse_setattr_in {
uint32_t valid;
uint32_t padding;
uint64_t fh;
uint64_t size;
uint64_t lock_owner;
uint64_t atime;
uint64_t mtime;
uint64_t ctime;
uint32_t atimensec;
uint32_t mtimensec;
uint32_t ctimensec;
uint32_t mode;
uint32_t unused4;
uint32_t uid;
uint32_t gid;
uint32_t unused5;
};
struct fuse_open_in {
uint32_t flags;
uint32_t unused;
};
struct fuse_create_in {
uint32_t flags;
uint32_t mode;
uint32_t umask;
uint32_t padding;
};
struct fuse_open_out {
uint64_t fh;
uint32_t open_flags;
uint32_t padding;
};
struct fuse_release_in {
uint64_t fh;
uint32_t flags;
uint32_t release_flags;
uint64_t lock_owner;
};
struct fuse_flush_in {
uint64_t fh;
uint32_t unused;
uint32_t padding;
uint64_t lock_owner;
};
struct fuse_read_in {
uint64_t fh;
uint64_t offset;
uint32_t size;
uint32_t read_flags;
uint64_t lock_owner;
uint32_t flags;
uint32_t padding;
};
#define FUSE_COMPAT_WRITE_IN_SIZE 24
struct fuse_write_in {
uint64_t fh;
uint64_t offset;
uint32_t size;
uint32_t write_flags;
uint64_t lock_owner;
uint32_t flags;
uint32_t padding;
};
struct fuse_write_out {
uint32_t size;
uint32_t padding;
};
#define FUSE_COMPAT_STATFS_SIZE 48
struct fuse_statfs_out {
struct fuse_kstatfs st;
};
struct fuse_fsync_in {
uint64_t fh;
uint32_t fsync_flags;
uint32_t padding;
};
struct fuse_setxattr_in {
uint32_t size;
uint32_t flags;
};
struct fuse_getxattr_in {
uint32_t size;
uint32_t padding;
};
struct fuse_getxattr_out {
uint32_t size;
uint32_t padding;
};
struct fuse_lk_in {
uint64_t fh;
uint64_t owner;
struct fuse_file_lock lk;
uint32_t lk_flags;
uint32_t padding;
};
struct fuse_lk_out {
struct fuse_file_lock lk;
};
struct fuse_access_in {
uint32_t mask;
uint32_t padding;
};
struct fuse_init_in {
uint32_t major;
uint32_t minor;
uint32_t max_readahead;
uint32_t flags;
};
#define FUSE_COMPAT_INIT_OUT_SIZE 8
#define FUSE_COMPAT_22_INIT_OUT_SIZE 24
struct fuse_init_out {
uint32_t major;
uint32_t minor;
uint32_t max_readahead;
uint32_t flags;
uint16_t max_background;
uint16_t congestion_threshold;
uint32_t max_write;
uint32_t time_gran;
uint16_t max_pages;
uint16_t padding;
uint32_t unused[8];
};
#define CUSE_INIT_INFO_MAX 4096
struct cuse_init_in {
uint32_t major;
uint32_t minor;
uint32_t unused;
uint32_t flags;
};
struct cuse_init_out {
uint32_t major;
uint32_t minor;
uint32_t unused;
uint32_t flags;
uint32_t max_read;
uint32_t max_write;
uint32_t dev_major; /* chardev major */
uint32_t dev_minor; /* chardev minor */
uint32_t spare[10];
};
struct fuse_interrupt_in {
uint64_t unique;
};
struct fuse_bmap_in {
uint64_t block;
uint32_t blocksize;
uint32_t padding;
};
struct fuse_bmap_out {
uint64_t block;
};
struct fuse_ioctl_in {
uint64_t fh;
uint32_t flags;
uint32_t cmd;
uint64_t arg;
uint32_t in_size;
uint32_t out_size;
};
struct fuse_ioctl_iovec {
uint64_t base;
uint64_t len;
};
struct fuse_ioctl_out {
int32_t result;
uint32_t flags;
uint32_t in_iovs;
uint32_t out_iovs;
};
struct fuse_poll_in {
uint64_t fh;
uint64_t kh;
uint32_t flags;
uint32_t events;
};
struct fuse_poll_out {
uint32_t revents;
uint32_t padding;
};
struct fuse_notify_poll_wakeup_out {
uint64_t kh;
};
struct fuse_fallocate_in {
uint64_t fh;
uint64_t offset;
uint64_t length;
uint32_t mode;
uint32_t padding;
};
struct fuse_in_header {
uint32_t len;
uint32_t opcode;
uint64_t unique;
uint64_t nodeid;
uint32_t uid;
uint32_t gid;
uint32_t pid;
uint32_t padding;
};
struct fuse_out_header {
uint32_t len;
int32_t error;
uint64_t unique;
};
struct fuse_dirent {
uint64_t ino;
uint64_t off;
uint32_t namelen;
uint32_t type;
char name[];
};
#define FUSE_NAME_OFFSET offsetof(struct fuse_dirent, name)
#define FUSE_DIRENT_ALIGN(x) \
(((x) + sizeof(uint64_t) - 1) & ~(sizeof(uint64_t) - 1))
#define FUSE_DIRENT_SIZE(d) \
FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET + (d)->namelen)
struct fuse_direntplus {
struct fuse_entry_out entry_out;
struct fuse_dirent dirent;
};
#define FUSE_NAME_OFFSET_DIRENTPLUS \
offsetof(struct fuse_direntplus, dirent.name)
#define FUSE_DIRENTPLUS_SIZE(d) \
FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET_DIRENTPLUS + (d)->dirent.namelen)
struct fuse_notify_inval_inode_out {
uint64_t ino;
int64_t off;
int64_t len;
};
struct fuse_notify_inval_entry_out {
uint64_t parent;
uint32_t namelen;
uint32_t padding;
};
struct fuse_notify_delete_out {
uint64_t parent;
uint64_t child;
uint32_t namelen;
uint32_t padding;
};
struct fuse_notify_store_out {
uint64_t nodeid;
uint64_t offset;
uint32_t size;
uint32_t padding;
};
struct fuse_notify_retrieve_out {
uint64_t notify_unique;
uint64_t nodeid;
uint64_t offset;
uint32_t size;
uint32_t padding;
};
/* Matches the size of fuse_write_in */
struct fuse_notify_retrieve_in {
uint64_t dummy1;
uint64_t offset;
uint32_t size;
uint32_t dummy2;
uint64_t dummy3;
uint64_t dummy4;
};
/* Device ioctls: */
#define FUSE_DEV_IOC_CLONE _IOR(229, 0, uint32_t)
struct fuse_lseek_in {
uint64_t fh;
uint64_t offset;
uint32_t whence;
uint32_t padding;
};
struct fuse_lseek_out {
uint64_t offset;
};
struct fuse_copy_file_range_in {
uint64_t fh_in;
uint64_t off_in;
uint64_t nodeid_out;
uint64_t fh_out;
uint64_t off_out;
uint64_t len;
uint64_t flags;
};
#endif /* _LINUX_FUSE_H */

1023
fuseuring_main.cpp Executable file
View File

File diff suppressed because it is too large Load Diff

7
fuseuring_main.h Executable file
View File

@ -0,0 +1,7 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// Copyright (C) Martin Raiber
#pragma once
#include "../Interface/File.h"
#include <string>
int fuseuring_main(int backing_fd, const std::string& mountpoint, int max_background, int congestion_threshold);

59
main.cpp Executable file
View File

@ -0,0 +1,59 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// Copyright (C) Martin Raiber
#include "fuseuring_main.h"
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <iostream>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/mman.h>
#include <stdlib.h>
int main(int argc, char* argv[])
{
if(argc<5)
{
std::cerr << "Not enough arguments ./fuseuring [backing file path] [fuse mount path] [backing file size] [fuse max_background]" << std::endl;
return 101;
}
int backing_fd = open(argv[1], O_CLOEXEC|O_CREAT|O_RDWR);
//int backing_fd = memfd_create("backing_file", MFD_CLOEXEC);
if(backing_fd==-1)
{
perror("Error opening backing file");
return 1;
}
int64_t backing_file_size = atoll(argv[3]);
int rc = posix_fallocate(backing_fd, 0, backing_file_size);
if(rc!=0)
{
std::cerr << "Error allocating 1GB for backing file rc: " << rc << std::endl;
return 1;
}
rc = prctl(PR_SET_IO_FLUSHER, 1, 0, 0, 0);
if(rc!=0)
{
perror("Error setting PR_SET_IO_FLUSHER");
}
struct rlimit rlimit;
rlimit.rlim_cur = RLIM_INFINITY;
rlimit.rlim_max = RLIM_INFINITY;
rc = setrlimit(RLIMIT_MEMLOCK, &rlimit);
if(rc!=0)
{
perror("Error increasing RLIMIT_MEMLOCK");
}
int fuse_max_background = atoi(argv[4]);
return fuseuring_main(backing_fd, argv[2], fuse_max_background, fuse_max_background+1000);
}

149
readme.md Normal file
View File

@ -0,0 +1,149 @@
## Fuseuring
This is an example program that demonstrates how to use io_uring to drive Linux fuse. The only thing it does is forward a single file to the fuse mount point.
### Why?
Using io_uring reduces the number of system calls the fuse program makes which should speed it up.
Fuse is sometimes used to export a single file to be used as volume, as well. E.g. vdfuse, [s3backer](https://github.com/archiecobbs/s3backer), [UrBackup](https://www.urbackup.org/) (vhd/vhdz mounting). Recent improvements in loop (direct-io), fuse and Linux memory management (`PR_SET_IO_FLUSHER`) have made this really performant.
### Performance
With large iodepth it gets good results (backing file on tmpfs):
```
fio ~/fio/ssd-test.fio
seq-read: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1024
rand-read: (g=1): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1024
seq-write: (g=2): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1024
rand-write: (g=3): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1024
fio-3.21
Starting 4 processes
Jobs: 1 (f=1): [_(3),w(1)][100.0%][w=384MiB/s][w=98.3k IOPS][eta 00m:00s]
seq-read: (groupid=0, jobs=1): err= 0: pid=178089: Sat Nov 7 16:39:44 2020
read: IOPS=165k, BW=643MiB/s (674MB/s)(3500MiB/5443msec)
slat (nsec): min=1000, max=9680.6k, avg=4407.18, stdev=18980.57
clat (usec): min=318, max=33373, avg=6190.66, stdev=3064.92
lat (usec): min=320, max=33379, avg=6195.14, stdev=3066.45
clat percentiles (usec):
| 1.00th=[ 1778], 5.00th=[ 2573], 10.00th=[ 2999], 20.00th=[ 3621],
| 30.00th=[ 4228], 40.00th=[ 4817], 50.00th=[ 5473], 60.00th=[ 6259],
| 70.00th=[ 7308], 80.00th=[ 8586], 90.00th=[10421], 95.00th=[11994],
| 99.00th=[14222], 99.50th=[15401], 99.90th=[26608], 99.95th=[28967],
| 99.99th=[32637]
bw ( KiB/s): min=614544, max=745196, per=100.00%, avg=662270.43, stdev=49887.45, samples=7
iops : min=153636, max=186299, avg=165567.43, stdev=12471.72, samples=7
lat (usec) : 500=0.01%, 750=0.02%, 1000=0.06%
lat (msec) : 2=1.54%, 4=24.67%, 10=61.52%, 20=11.89%, 50=0.30%
cpu : usr=18.83%, sys=43.37%, ctx=84144, majf=0, minf=1036
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=896000,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1024
rand-read: (groupid=1, jobs=1): err= 0: pid=178096: Sat Nov 7 16:39:44 2020
read: IOPS=91.0k, BW=356MiB/s (373MB/s)(3500MiB/9845msec)
slat (nsec): min=1100, max=16238k, avg=9354.69, stdev=93603.08
clat (usec): min=189, max=47250, avg=11192.69, stdev=2888.77
lat (usec): min=192, max=47252, avg=11202.13, stdev=2890.28
clat percentiles (usec):
| 1.00th=[ 6521], 5.00th=[ 7635], 10.00th=[ 8225], 20.00th=[ 8979],
| 30.00th=[ 9634], 40.00th=[10290], 50.00th=[10814], 60.00th=[11469],
| 70.00th=[12125], 80.00th=[13042], 90.00th=[14353], 95.00th=[15664],
| 99.00th=[20317], 99.50th=[22938], 99.90th=[34866], 99.95th=[42730],
| 99.99th=[44303]
bw ( KiB/s): min=333610, max=383504, per=100.00%, avg=365092.00, stdev=16877.24, samples=13
iops : min=83402, max=95876, avg=91272.69, stdev=4219.28, samples=13
lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.06%, 4=0.16%, 10=34.80%, 20=63.89%, 50=1.06%
cpu : usr=11.62%, sys=27.98%, ctx=128214, majf=0, minf=1037
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=896000,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1024
seq-write: (groupid=2, jobs=1): err= 0: pid=178107: Sat Nov 7 16:39:44 2020
write: IOPS=131k, BW=513MiB/s (538MB/s)(3500MiB/6818msec); 0 zone resets
slat (nsec): min=1100, max=4008.6k, avg=5821.89, stdev=18205.65
clat (usec): min=239, max=21808, avg=7749.25, stdev=3409.43
lat (usec): min=243, max=21809, avg=7755.16, stdev=3411.55
clat percentiles (usec):
| 1.00th=[ 2638], 5.00th=[ 3163], 10.00th=[ 3654], 20.00th=[ 4555],
| 30.00th=[ 5407], 40.00th=[ 6390], 50.00th=[ 7373], 60.00th=[ 8356],
| 70.00th=[ 9372], 80.00th=[10683], 90.00th=[12256], 95.00th=[14091],
| 99.00th=[17171], 99.50th=[17957], 99.90th=[19792], 99.95th=[20317],
| 99.99th=[21103]
bw ( KiB/s): min=479727, max=612688, per=99.28%, avg=521884.56, stdev=40381.06, samples=9
iops : min=119931, max=153172, avg=130470.78, stdev=10095.48, samples=9
lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.16%, 4=13.54%, 10=61.47%, 20=24.72%, 50=0.08%
cpu : usr=17.10%, sys=39.93%, ctx=112618, majf=0, minf=14
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,896000,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1024
rand-write: (groupid=3, jobs=1): err= 0: pid=178115: Sat Nov 7 16:39:44 2020
write: IOPS=103k, BW=403MiB/s (422MB/s)(3500MiB/8695msec); 0 zone resets
slat (nsec): min=1200, max=16351k, avg=8084.77, stdev=67434.88
clat (usec): min=1126, max=38086, avg=9877.97, stdev=2952.08
lat (usec): min=1128, max=38088, avg=9886.14, stdev=2953.72
clat percentiles (usec):
| 1.00th=[ 6325], 5.00th=[ 7373], 10.00th=[ 7701], 20.00th=[ 8094],
| 30.00th=[ 8455], 40.00th=[ 8848], 50.00th=[ 9110], 60.00th=[ 9503],
| 70.00th=[10159], 80.00th=[11076], 90.00th=[12518], 95.00th=[14615],
| 99.00th=[23725], 99.50th=[26870], 99.90th=[32637], 99.95th=[35390],
| 99.99th=[36963]
bw ( KiB/s): min=340472, max=447488, per=99.79%, avg=411309.00, stdev=31900.28, samples=17
iops : min=85118, max=111872, avg=102827.24, stdev=7975.05, samples=17
lat (msec) : 2=0.04%, 4=0.28%, 10=67.14%, 20=30.73%, 50=1.82%
cpu : usr=13.52%, sys=30.96%, ctx=74227, majf=0, minf=12
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,896000,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1024
Run status group 0 (all jobs):
READ: bw=643MiB/s (674MB/s), 643MiB/s-643MiB/s (674MB/s-674MB/s), io=3500MiB (3670MB), run=5443-5443msec
Run status group 1 (all jobs):
READ: bw=356MiB/s (373MB/s), 356MiB/s-356MiB/s (373MB/s-373MB/s), io=3500MiB (3670MB), run=9845-9845msec
Run status group 2 (all jobs):
WRITE: bw=513MiB/s (538MB/s), 513MiB/s-513MiB/s (538MB/s-538MB/s), io=3500MiB (3670MB), run=6818-6818msec
Run status group 3 (all jobs):
WRITE: bw=403MiB/s (422MB/s), 403MiB/s-403MiB/s (422MB/s-422MB/s), io=3500MiB (3670MB), run=8695-8695msec
Disk stats (read/write):
loop0: ios=1363439/1466332, merge=428561/321891, ticks=1868225/1741830, in_queue=3610054, util=94.74%
```
### How to compile
Need gcc >= 10 for C++ coroutines. Depends on (recent) liburing-dev.
```bash
autoreconf --install
./configure
make
```
### How to run
Needs a recent Linux kernel (>=5.9) for io_uring functionality and recent `losetup`.
```bash
FMNT=/media/test
BMNT=/media/bench
mkdir -p "$FMNT"
./fuseuring /tmp/backing_file.img "$FMNT" $((500*1024*1024)) 5000 &
LODEV=$(losetup --find --show "$FMNT/volume" --direct-io=on)
mkfs.ext4 $LODEV || true
mount $LODEV "$BMNT"
losetup -d $LODEV
```
Or see `bench.sh`.