Skip to content

Speed-up waker by using uninitialized array #4055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Aug 25, 2021
Merged

Speed-up waker by using uninitialized array #4055

Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7fd2ae7
Use uninitialized array in wake0 to prevent initialiation of each Opt…
Aug 23, 2021
01b91ab
avoid using Option
Aug 23, 2021
2bb9da5
rustfmt
Aug 23, 2021
c0aad94
implement WakeList to support Drop on it
Aug 23, 2021
b932ea5
don't return from push
Aug 23, 2021
03103d8
Prevent later dropping of uninitialized Waker if `wake` panics.
glebpom Aug 23, 2021
9dd73eb
Fix `can_push` condition - allow usage of the last waker.
Aug 23, 2021
0fb5f46
Use slice_from_raw_parts_mut/drop_in_place in WakerList Drop implemen…
glebpom Aug 23, 2021
c7e65e6
remove unused import
Aug 23, 2021
5040110
fmt
Aug 23, 2021
7068c3e
try to fix imports
Aug 23, 2021
958d43b
clippy
Aug 23, 2021
7a55899
import core::ptr
Aug 23, 2021
905dc93
use for loop with rev iterator in wake_all
Aug 23, 2021
cbdcfd3
Revert "use for loop with rev iterator in wake_all"
Aug 23, 2021
0267a17
rustfmt
Aug 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions tokio/src/io/driver/scheduled_io.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::Mutex;
use crate::util::bit;
use crate::util::slab::Entry;
use core::mem::MaybeUninit;

use std::sync::atomic::Ordering::{AcqRel, Acquire, Release};
use std::task::{Context, Poll, Waker};
Expand Down Expand Up @@ -214,7 +215,10 @@ impl ScheduledIo {
fn wake0(&self, ready: Ready, shutdown: bool) {
const NUM_WAKERS: usize = 32;

let mut wakers: [Option<Waker>; NUM_WAKERS] = Default::default();
let mut wakers: [MaybeUninit<Option<Waker>>; NUM_WAKERS] = unsafe {
core::mem::MaybeUninit::uninit().assume_init()
};

let mut curr = 0;

let mut waiters = self.waiters.lock();
Expand All @@ -224,15 +228,15 @@ impl ScheduledIo {
// check for AsyncRead slot
if ready.is_readable() {
if let Some(waker) = waiters.reader.take() {
wakers[curr] = Some(waker);
wakers[curr] = MaybeUninit::new(Some(waker));
curr += 1;
}
}

// check for AsyncWrite slot
if ready.is_writable() {
if let Some(waker) = waiters.writer.take() {
wakers[curr] = Some(waker);
wakers[curr] = MaybeUninit::new(Some(waker));
curr += 1;
}
}
Expand All @@ -248,7 +252,7 @@ impl ScheduledIo {

if let Some(waker) = waiter.waker.take() {
waiter.is_ready = true;
wakers[curr] = Some(waker);
wakers[curr] = MaybeUninit::new(Some(waker));
curr += 1;
}
}
Expand All @@ -260,8 +264,8 @@ impl ScheduledIo {

drop(waiters);

for waker in wakers.iter_mut().take(curr) {
waker.take().unwrap().wake();
for waker in &mut wakers[..curr] {
unsafe { &mut *waker.as_mut_ptr() }.take().unwrap().wake();
}

curr = 0;
Expand All @@ -273,8 +277,8 @@ impl ScheduledIo {
// Release the lock before notifying
drop(waiters);

for waker in wakers.iter_mut().take(curr) {
waker.take().unwrap().wake();
for waker in &mut wakers[..curr] {
unsafe { &mut *waker.as_mut_ptr() }.take().unwrap().wake();
}
}

Expand Down