parallel.rs - source (original) (raw)
rustc_data_structures/sync/
parallel.rs
1//! This module defines parallel operations that are implemented in
2//! one way for the serial compiler, and another way the parallel compiler.
3
4use std::any::Any;
5use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
6
7use parking_lot::Mutex;
8
9use crate::FatalErrorMarker;
10use crate::sync::{DynSend, DynSync, FromDyn, IntoDynSyncSend, mode};
11
12/// A guard used to hold panics that occur during a parallel section to later by unwound.
13/// This is used for the parallel compiler to prevent fatal errors from non-deterministically
14/// hiding errors by ensuring that everything in the section has completed executing before
15/// continuing with unwinding. It's also used for the non-parallel code to ensure error message
16/// output match the parallel compiler for testing purposes.
17pub struct ParallelGuard {
18 panic: Mutex<Option<IntoDynSyncSend<Box<dyn Any + Send + 'static>>>>,
19}
20
21impl ParallelGuard {
22 pub fn run<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
23 catch_unwind(AssertUnwindSafe(f))
24 .map_err(|err| {
25 let mut panic = self.panic.lock();
26 if panic.is_none() || !(*err).is::<FatalErrorMarker>() {
27 *panic = Some(IntoDynSyncSend(err));
28 }
29 })
30 .ok()
31 }
32}
33
34/// This gives access to a fresh parallel guard in the closure and will unwind any panics
35/// caught in it after the closure returns.
36#[inline]
37pub fn parallel_guard<R>(f: impl FnOnce(&ParallelGuard) -> R) -> R {
38 let guard = ParallelGuard { panic: Mutex::new(None) };
39 let ret = f(&guard);
40 if let Some(IntoDynSyncSend(panic)) = guard.panic.into_inner() {
41 resume_unwind(panic);
42 }
43 ret
44}
45
46fn serial_join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
47where
48 A: FnOnce() -> RA,
49 B: FnOnce() -> RB,
50{
51 let (a, b) = parallel_guard(|guard| {
52 let a = guard.run(oper_a);
53 let b = guard.run(oper_b);
54 (a, b)
55 });
56 (a.unwrap(), b.unwrap())
57}
58
59/// Runs a list of blocks in parallel. The first block is executed immediately on
60/// the current thread. Use that for the longest running block.
61#[macro_export]
62macro_rules! parallel {
63 (impl <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>f</mi><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo>:</mo><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo stretchy="false">[</mo></mrow><annotation encoding="application/x-tex">fblock:block [</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mopen">[</span></span></span></span>($c:expr,)*] [$block:expr <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo><mo separator="true">,</mo></mrow><annotation encoding="application/x-tex">(, </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mpunct">,</span></span></span></span>rest:expr)*]) => {
64 parallel!(impl <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>f</mi><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo stretchy="false">[</mo></mrow><annotation encoding="application/x-tex">fblock [</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mopen">[</span></span></span></span>block, <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo></mrow><annotation encoding="application/x-tex">(</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span></span></span></span>c,)*] [$($rest),*])
65 };
66 (impl <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>f</mi><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo>:</mo><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo stretchy="false">[</mo></mrow><annotation encoding="application/x-tex">fblock:block [</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">:</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mopen">[</span></span></span></span>($blocks:expr,)*] []) => {
67 $crate::sync::parallel_guard(|guard| {
68 $crate::sync::scope(|s| {
69 $(
70 let block = <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>c</mi><mi>r</mi><mi>a</mi><mi>t</mi><mi>e</mi><mo>:</mo><mo>:</mo><mi>s</mi><mi>y</mi><mi>n</mi><mi>c</mi><mo>:</mo><mo>:</mo><mi>F</mi><mi>r</mi><mi>o</mi><mi>m</mi><mi>D</mi><mi>y</mi><mi>n</mi><mo>:</mo><mo>:</mo><mi>f</mi><mi>r</mi><mi>o</mi><mi>m</mi><mo stretchy="false">(</mo><mi mathvariant="normal">∣</mi><mi mathvariant="normal">∣</mi></mrow><annotation encoding="application/x-tex">crate::sync::FromDyn::from(|| </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6151em;"></span><span class="mord mathnormal" style="margin-right:0.02778em;">cr</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">e</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">::</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">sy</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">::</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.8778em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">F</span><span class="mord mathnormal">ro</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.03588em;">Dy</span><span class="mord mathnormal">n</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">::</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mord mathnormal">ro</span><span class="mord mathnormal">m</span><span class="mopen">(</span><span class="mord">∣∣</span></span></span></span>blocks);
71 s.spawn(move |_| {
72 guard.run(move || block.into_inner()());
73 });
74 )*
75 guard.run(|| $fblock);
76 });
77 });
78 };
79 ($fblock:block, <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo></mrow><annotation encoding="application/x-tex">(</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span></span></span></span>blocks:block),*) => {
80 if $crate::sync::is_dyn_thread_safe() {
81 // Reverse the order of the later blocks since Rayon executes them in reverse order
82 // when using a single thread. This ensures the execution order matches that
83 // of a single threaded rustc.
84 parallel!(impl <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>f</mi><mi>b</mi><mi>l</mi><mi>o</mi><mi>c</mi><mi>k</mi><mo stretchy="false">[</mo><mo stretchy="false">]</mo><mo stretchy="false">[</mo></mrow><annotation encoding="application/x-tex">fblock [] [</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mord mathnormal">b</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord mathnormal">oc</span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span><span class="mopen">[</span><span class="mclose">]</span><span class="mopen">[</span></span></span></span>($blocks),*]);
85 } else {
86 $crate::sync::parallel_guard(|guard| {
87 guard.run(|| $fblock);
88 <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">(</mo><mi>g</mi><mi>u</mi><mi>a</mi><mi>r</mi><mi>d</mi><mi mathvariant="normal">.</mi><mi>r</mi><mi>u</mi><mi>n</mi><mo stretchy="false">(</mo><mi mathvariant="normal">∣</mi><mi mathvariant="normal">∣</mi></mrow><annotation encoding="application/x-tex">(guard.run(|| </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mopen">(</span><span class="mord mathnormal">gu</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="mord mathnormal">d</span><span class="mord">.</span><span class="mord mathnormal" style="margin-right:0.02778em;">r</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord">∣∣</span></span></span></span>blocks);)*
89 });
90 }
91 };
92 }
93
94pub fn spawn(func: impl FnOnce() + DynSend + 'static) {
95 if mode::is_dyn_thread_safe() {
96 let func = FromDyn::from(func);
97 rustc_thread_pool::spawn(|| {
98 (func.into_inner())();
99 });
100 } else {
101 func()
102 }
103}
104
105// This function only works when `mode::is_dyn_thread_safe()`.
106pub fn scope<'scope, OP, R>(op: OP) -> R
107where
108 OP: FnOnce(&rustc_thread_pool::Scope<'scope>) -> R + DynSend,
109 R: DynSend,
110{
111 let op = FromDyn::from(op);
112 rustc_thread_pool::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
113}
114
115#[inline]
116pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
117where
118 A: FnOnce() -> RA + DynSend,
119 B: FnOnce() -> RB + DynSend,
120{
121 if mode::is_dyn_thread_safe() {
122 let oper_a = FromDyn::from(oper_a);
123 let oper_b = FromDyn::from(oper_b);
124 let (a, b) = parallel_guard(|guard| {
125 rustc_thread_pool::join(
126 move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
127 move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
128 )
129 });
130 (a.unwrap().into_inner(), b.unwrap().into_inner())
131 } else {
132 serial_join(oper_a, oper_b)
133 }
134}
135
136fn par_slice<I: DynSend>(
137 items: &mut [I],
138 guard: &ParallelGuard,
139 for_each: impl Fn(&mut I) + DynSync + DynSend,
140) {
141 struct State<'a, F> {
142 for_each: FromDyn<F>,
143 guard: &'a ParallelGuard,
144 group: usize,
145 }
146
147 fn par_rec<I: DynSend, F: Fn(&mut I) + DynSync + DynSend>(
148 items: &mut [I],
149 state: &State<'_, F>,
150 ) {
151 if items.len() <= state.group {
152 for item in items {
153 state.guard.run(|| (state.for_each)(item));
154 }
155 } else {
156 let (left, right) = items.split_at_mut(items.len() / 2);
157 let mut left = state.for_each.derive(left);
158 let mut right = state.for_each.derive(right);
159 rustc_thread_pool::join(move || par_rec(*left, state), move || par_rec(*right, state));
160 }
161 }
162
163 let state = State {
164 for_each: FromDyn::from(for_each),
165 guard,
166 group: std::cmp::max(items.len() / 128, 1),
167 };
168 par_rec(items, &state)
169}
170
171pub fn par_for_each_in<I: DynSend, T: IntoIterator<Item = I>>(
172 t: T,
173 for_each: impl Fn(&I) + DynSync + DynSend,
174) {
175 parallel_guard(|guard| {
176 if mode::is_dyn_thread_safe() {
177 let mut items: Vec<_> = t.into_iter().collect();
178 par_slice(&mut items, guard, |i| for_each(&*i))
179 } else {
180 t.into_iter().for_each(|i| {
181 guard.run(|| for_each(&i));
182 });
183 }
184 });
185}
186
187/// This runs `for_each` in parallel for each iterator item. If one or more of the
188/// `for_each` calls returns `Err`, the function will also return `Err`. The error returned
189/// will be non-deterministic, but this is expected to be used with `ErrorGuaranteed` which
190/// are all equivalent.
191pub fn try_par_for_each_in<T: IntoIterator, E: DynSend>(
192 t: T,
193 for_each: impl Fn(&<T as IntoIterator>::Item) -> Result<(), E> + DynSync + DynSend,
194) -> Result<(), E>
195where
196 <T as IntoIterator>::Item: DynSend,
197{
198 parallel_guard(|guard| {
199 if mode::is_dyn_thread_safe() {
200 let mut items: Vec<_> = t.into_iter().collect();
201
202 let error = Mutex::new(None);
203
204 par_slice(&mut items, guard, |i| {
205 if let Err(err) = for_each(&*i) {
206 *error.lock() = Some(err);
207 }
208 });
209
210 if let Some(err) = error.into_inner() { Err(err) } else { Ok(()) }
211 } else {
212 t.into_iter().filter_map(|i| guard.run(|| for_each(&i))).fold(Ok(()), Result::and)
213 }
214 })
215}
216
217pub fn par_map<I: DynSend, T: IntoIterator<Item = I>, R: DynSend, C: FromIterator<R>>(
218 t: T,
219 map: impl Fn(I) -> R + DynSync + DynSend,
220) -> C {
221 parallel_guard(|guard| {
222 if mode::is_dyn_thread_safe() {
223 let map = FromDyn::from(map);
224
225 let mut items: Vec<(Option<I>, Option<R>)> =
226 t.into_iter().map(|i| (Some(i), None)).collect();
227
228 par_slice(&mut items, guard, |i| {
229 i.1 = Some(map(i.0.take().unwrap()));
230 });
231
232 items.into_iter().filter_map(|i| i.1).collect()
233 } else {
234 t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()
235 }
236 })
237}
238
239pub fn broadcast<R: DynSend>(op: impl Fn(usize) -> R + DynSync) -> Vec<R> {
240 if mode::is_dyn_thread_safe() {
241 let op = FromDyn::from(op);
242 let results = rustc_thread_pool::broadcast(|context| op.derive(op(context.index())));
243 results.into_iter().map(|r| r.into_inner()).collect()
244 } else {
245 vec![op(0)]
246 }
247}