make.rs - source (original) (raw)
rustdoc/doctest/
make.rs
1//! Logic for transforming the raw code given by the user into something actually
2//! runnable, e.g. by adding a `main` function if it doesn't already exist.
3
4use std::fmt::{self, Write as _};
5use std::io;
6use std::sync::Arc;
7
8use rustc_ast::token::{Delimiter, TokenKind};
9use rustc_ast::tokenstream::TokenTree;
10use rustc_ast::{self as ast, AttrStyle, HasAttrs, StmtKind};
11use rustc_errors::emitter::stderr_destination;
12use rustc_errors::{ColorConfig, DiagCtxtHandle};
13use rustc_parse::new_parser_from_source_str;
14use rustc_session::parse::ParseSess;
15use rustc_span::edition::{DEFAULT_EDITION, Edition};
16use rustc_span::source_map::SourceMap;
17use rustc_span::symbol::sym;
18use rustc_span::{DUMMY_SP, FileName, Span, kw};
19use tracing::debug;
20
21use super::GlobalTestOptions;
22use crate::display::Joined as _;
23use crate::html::markdown::LangString;
24
25#[derive(Default)]
26struct ParseSourceInfo {
27 has_main_fn: bool,
28 already_has_extern_crate: bool,
29 supports_color: bool,
30 has_global_allocator: bool,
31 has_macro_def: bool,
32 everything_else: String,
33 crates: String,
34 crate_attrs: String,
35 maybe_crate_attrs: String,
36}
37
38/// Builder type for `DocTestBuilder`.
39pub(crate) struct BuildDocTestBuilder<'a> {
40 source: &'a str,
41 crate_name: Option<&'a str>,
42 edition: Edition,
43 can_merge_doctests: bool,
44 // If `test_id` is `None`, it means we're generating code for a code example "run" link.
45 test_id: Option<String>,
46 lang_str: Option<&'a LangString>,
47 span: Span,
48}
49
50impl<'a> BuildDocTestBuilder<'a> {
51 pub(crate) fn new(source: &'a str) -> Self {
52 Self {
53 source,
54 crate_name: None,
55 edition: DEFAULT_EDITION,
56 can_merge_doctests: false,
57 test_id: None,
58 lang_str: None,
59 span: DUMMY_SP,
60 }
61 }
62
63 #[inline]
64 pub(crate) fn crate_name(mut self, crate_name: &'a str) -> Self {
65 self.crate_name = Some(crate_name);
66 self
67 }
68
69 #[inline]
70 pub(crate) fn can_merge_doctests(mut self, can_merge_doctests: bool) -> Self {
71 self.can_merge_doctests = can_merge_doctests;
72 self
73 }
74
75 #[inline]
76 pub(crate) fn test_id(mut self, test_id: String) -> Self {
77 self.test_id = Some(test_id);
78 self
79 }
80
81 #[inline]
82 pub(crate) fn lang_str(mut self, lang_str: &'a LangString) -> Self {
83 self.lang_str = Some(lang_str);
84 self
85 }
86
87 #[inline]
88 pub(crate) fn span(mut self, span: Span) -> Self {
89 self.span = span;
90 self
91 }
92
93 #[inline]
94 pub(crate) fn edition(mut self, edition: Edition) -> Self {
95 self.edition = edition;
96 self
97 }
98
99 pub(crate) fn build(self, dcx: Option<DiagCtxtHandle<'_>>) -> DocTestBuilder {
100 let BuildDocTestBuilder {
101 source,
102 crate_name,
103 edition,
104 can_merge_doctests,
105 // If `test_id` is `None`, it means we're generating code for a code example "run" link.
106 test_id,
107 lang_str,
108 span,
109 } = self;
110 let can_merge_doctests = can_merge_doctests
111 && lang_str.is_some_and(|lang_str| {
112 !lang_str.compile_fail && !lang_str.test_harness && !lang_str.standalone_crate
113 });
114
115 let result = rustc_driver::catch_fatal_errors(|| {
116 rustc_span::create_session_if_not_set_then(edition, |_| {
117 parse_source(source, &crate_name, dcx, span)
118 })
119 });
120
121 let Ok(Ok(ParseSourceInfo {
122 has_main_fn,
123 already_has_extern_crate,
124 supports_color,
125 has_global_allocator,
126 has_macro_def,
127 everything_else,
128 crates,
129 crate_attrs,
130 maybe_crate_attrs,
131 })) = result
132 else {
133 // If the AST returned an error, we don't want this doctest to be merged with the
134 // others.
135 return DocTestBuilder::invalid(
136 String::new(),
137 String::new(),
138 String::new(),
139 source.to_string(),
140 test_id,
141 );
142 };
143
144 debug!("crate_attrs:\n{crate_attrs}{maybe_crate_attrs}");
145 debug!("crates:\n{crates}");
146 debug!("after:\n{everything_else}");
147
148 // If it contains `#[feature]` or `#[no_std]`, we don't want it to be merged either.
149 let can_be_merged = can_merge_doctests
150 && !has_global_allocator
151 && crate_attrs.is_empty()
152 // If this is a merged doctest and a defined macro uses `$crate`, then the path will
153 // not work, so better not put it into merged doctests.
154 && !(has_macro_def && everything_else.contains("$crate"));
155 DocTestBuilder {
156 supports_color,
157 has_main_fn,
158 crate_attrs,
159 maybe_crate_attrs,
160 crates,
161 everything_else,
162 already_has_extern_crate,
163 test_id,
164 invalid_ast: false,
165 can_be_merged,
166 }
167 }
168}
169
170/// This struct contains information about the doctest itself which is then used to generate
171/// doctest source code appropriately.
172pub(crate) struct DocTestBuilder {
173 pub(crate) supports_color: bool,
174 pub(crate) already_has_extern_crate: bool,
175 pub(crate) has_main_fn: bool,
176 pub(crate) crate_attrs: String,
177 /// If this is a merged doctest, it will be put into `everything_else`, otherwise it will
178 /// put into `crate_attrs`.
179 pub(crate) maybe_crate_attrs: String,
180 pub(crate) crates: String,
181 pub(crate) everything_else: String,
182 pub(crate) test_id: Option<String>,
183 pub(crate) invalid_ast: bool,
184 pub(crate) can_be_merged: bool,
185}
186
187impl DocTestBuilder {
188 fn invalid(
189 crate_attrs: String,
190 maybe_crate_attrs: String,
191 crates: String,
192 everything_else: String,
193 test_id: Option<String>,
194 ) -> Self {
195 Self {
196 supports_color: false,
197 has_main_fn: false,
198 crate_attrs,
199 maybe_crate_attrs,
200 crates,
201 everything_else,
202 already_has_extern_crate: false,
203 test_id,
204 invalid_ast: true,
205 can_be_merged: false,
206 }
207 }
208
209 /// Transforms a test into code that can be compiled into a Rust binary, and returns the number of
210 /// lines before the test code begins.
211 pub(crate) fn generate_unique_doctest(
212 &self,
213 test_code: &str,
214 dont_insert_main: bool,
215 opts: &GlobalTestOptions,
216 crate_name: Option<&str>,
217 ) -> (String, usize) {
218 if self.invalid_ast {
219 // If the AST failed to compile, no need to go generate a complete doctest, the error
220 // will be better this way.
221 debug!("invalid AST:\n{test_code}");
222 return (test_code.to_string(), 0);
223 }
224 let mut line_offset = 0;
225 let mut prog = String::new();
226 let everything_else = self.everything_else.trim();
227 if opts.attrs.is_empty() {
228 // If there aren't any attributes supplied by #![doc(test(attr(...)))], then allow some
229 // lints that are commonly triggered in doctests. The crate-level test attributes are
230 // commonly used to make tests fail in case they trigger warnings, so having this there in
231 // that case may cause some tests to pass when they shouldn't have.
232 prog.push_str("#![allow(unused)]\n");
233 line_offset += 1;
234 }
235
236 // Next, any attributes that came from the crate root via #![doc(test(attr(...)))].
237 for attr in &opts.attrs {
238 prog.push_str(&format!("#![{attr}]\n"));
239 line_offset += 1;
240 }
241
242 // Now push any outer attributes from the example, assuming they
243 // are intended to be crate attributes.
244 if !self.crate_attrs.is_empty() {
245 prog.push_str(&self.crate_attrs);
246 if !self.crate_attrs.ends_with('\n') {
247 prog.push('\n');
248 }
249 }
250 if !self.maybe_crate_attrs.is_empty() {
251 prog.push_str(&self.maybe_crate_attrs);
252 if !self.maybe_crate_attrs.ends_with('\n') {
253 prog.push('\n');
254 }
255 }
256 if !self.crates.is_empty() {
257 prog.push_str(&self.crates);
258 if !self.crates.ends_with('\n') {
259 prog.push('\n');
260 }
261 }
262
263 // Don't inject `extern crate std` because it's already injected by the
264 // compiler.
265 if !self.already_has_extern_crate &&
266 !opts.no_crate_inject &&
267 let Some(crate_name) = crate_name &&
268 crate_name != "std" &&
269 // Don't inject `extern crate` if the crate is never used.
270 // NOTE: this is terribly inaccurate because it doesn't actually
271 // parse the source, but only has false positives, not false
272 // negatives.
273 test_code.contains(crate_name)
274 {
275 // rustdoc implicitly inserts an `extern crate` item for the own crate
276 // which may be unused, so we need to allow the lint.
277 prog.push_str("#[allow(unused_extern_crates)]\n");
278
279 prog.push_str(&format!("extern crate r#{crate_name};\n"));
280 line_offset += 1;
281 }
282
283 // FIXME: This code cannot yet handle no_std test cases yet
284 if dont_insert_main || self.has_main_fn || prog.contains("![no_std]") {
285 prog.push_str(everything_else);
286 } else {
287 let returns_result = everything_else.ends_with("(())");
288 // Give each doctest main function a unique name.
289 // This is for example needed for the tooling around `-C instrument-coverage`.
290 let inner_fn_name = if let Some(ref test_id) = self.test_id {
291 format!("_doctest_main_{test_id}")
292 } else {
293 "_inner".into()
294 };
295 let inner_attr = if self.test_id.is_some() { "#[allow(non_snake_case)] " } else { "" };
296 let (main_pre, main_post) = if returns_result {
297 (
298 format!(
299 "fn main() {{ {inner_attr}fn {inner_fn_name}() -> core::result::Result<(), impl core::fmt::Debug> {{\n",
300 ),
301 format!("\n}} {inner_fn_name}().unwrap() }}"),
302 )
303 } else if self.test_id.is_some() {
304 (
305 format!("fn main() {{ {inner_attr}fn {inner_fn_name}() {{\n",),
306 format!("\n}} {inner_fn_name}() }}"),
307 )
308 } else {
309 ("fn main() {\n".into(), "\n}".into())
310 };
311 // Note on newlines: We insert a line/newline *before*, and *after*
312 // the doctest and adjust the `line_offset` accordingly.
313 // In the case of `-C instrument-coverage`, this means that the generated
314 // inner `main` function spans from the doctest opening codeblock to the
315 // closing one. For example
316 // /// ``` <- start of the inner main
317 // /// <- code under doctest
318 // /// ``` <- end of the inner main
319 line_offset += 1;
320
321 prog.push_str(&main_pre);
322
323 // add extra 4 spaces for each line to offset the code block
324 if opts.insert_indent_space {
325 write!(
326 prog,
327 "{}",
328 fmt::from_fn(|f| everything_else
329 .lines()
330 .map(|line| fmt::from_fn(move |f| write!(f, " {line}")))
331 .joined("\n", f))
332 )
333 .unwrap();
334 } else {
335 prog.push_str(everything_else);
336 };
337 prog.push_str(&main_post);
338 }
339
340 debug!("final doctest:\n{prog}");
341
342 (prog, line_offset)
343 }
344}
345
346fn reset_error_count(psess: &ParseSess) {
347 // Reset errors so that they won't be reported as compiler bugs when dropping the
348 // dcx. Any errors in the tests will be reported when the test file is compiled,
349 // Note that we still need to cancel the errors above otherwise `Diag` will panic on
350 // drop.
351 psess.dcx().reset_err_count();
352}
353
354const DOCTEST_CODE_WRAPPER: &str = "fn f(){";
355
356fn parse_source(
357 source: &str,
358 crate_name: &Option<&str>,
359 parent_dcx: Option<DiagCtxtHandle<'_>>,
360 span: Span,
361) -> Result<ParseSourceInfo, ()> {
362 use rustc_errors::DiagCtxt;
363 use rustc_errors::emitter::{Emitter, HumanEmitter};
364 use rustc_span::source_map::FilePathMapping;
365
366 let mut info =
367 ParseSourceInfo { already_has_extern_crate: crate_name.is_none(), ..Default::default() };
368
369 let wrapped_source = format!("{DOCTEST_CODE_WRAPPER}{source}\n}}");
370
371 let filename = FileName::anon_source_code(&wrapped_source);
372
373 let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
374 let fallback_bundle = rustc_errors::fallback_fluent_bundle(
375 rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
376 false,
377 );
378 info.supports_color =
379 HumanEmitter::new(stderr_destination(ColorConfig::Auto), fallback_bundle.clone())
380 .supports_color();
381 // Any errors in parsing should also appear when the doctest is compiled for real, so just
382 // send all the errors that the parser emits directly into a `Sink` instead of stderr.
383 let emitter = HumanEmitter::new(Box::new(io::sink()), fallback_bundle);
384
385 // FIXME(misdreavus): pass `-Z treat-err-as-bug` to the doctest parser
386 let dcx = DiagCtxt::new(Box::new(emitter)).disable_warnings();
387 let psess = ParseSess::with_dcx(dcx, sm);
388
389 let mut parser = match new_parser_from_source_str(&psess, filename, wrapped_source) {
390 Ok(p) => p,
391 Err(errs) => {
392 errs.into_iter().for_each(|err| err.cancel());
393 reset_error_count(&psess);
394 return Err(());
395 }
396 };
397
398 fn push_to_s(s: &mut String, source: &str, span: rustc_span::Span, prev_span_hi: &mut usize) {
399 let extra_len = DOCTEST_CODE_WRAPPER.len();
400 // We need to shift by the length of `DOCTEST_CODE_WRAPPER` because we
401 // added it at the beginning of the source we provided to the parser.
402 let mut hi = span.hi().0 as usize - extra_len;
403 if hi > source.len() {
404 hi = source.len();
405 }
406 s.push_str(&source[*prev_span_hi..hi]);
407 *prev_span_hi = hi;
408 }
409
410 fn check_item(item: &ast::Item, info: &mut ParseSourceInfo, crate_name: &Option<&str>) -> bool {
411 let mut is_extern_crate = false;
412 if !info.has_global_allocator
413 && item.attrs.iter().any(|attr| attr.has_name(sym::global_allocator))
414 {
415 info.has_global_allocator = true;
416 }
417 match item.kind {
418 ast::ItemKind::Fn(ref fn_item) if !info.has_main_fn => {
419 if fn_item.ident.name == sym::main {
420 info.has_main_fn = true;
421 }
422 }
423 ast::ItemKind::ExternCrate(original, ident) => {
424 is_extern_crate = true;
425 if !info.already_has_extern_crate
426 && let Some(crate_name) = crate_name
427 {
428 info.already_has_extern_crate = match original {
429 Some(name) => name.as_str() == *crate_name,
430 None => ident.as_str() == *crate_name,
431 };
432 }
433 }
434 ast::ItemKind::MacroDef(..) => {
435 info.has_macro_def = true;
436 }
437 _ => {}
438 }
439 is_extern_crate
440 }
441
442 let mut prev_span_hi = 0;
443 let not_crate_attrs = &[sym::forbid, sym::allow, sym::warn, sym::deny, sym::expect];
444 let parsed = parser.parse_item(rustc_parse::parser::ForceCollect::No);
445
446 let result = match parsed {
447 Ok(Some(ref item))
448 if let ast::ItemKind::Fn(ref fn_item) = item.kind
449 && let Some(ref body) = fn_item.body =>
450 {
451 for attr in &item.attrs {
452 if attr.style == AttrStyle::Outer || attr.has_any_name(not_crate_attrs) {
453 // There is one exception to these attributes:
454 // `#![allow(internal_features)]`. If this attribute is used, we need to
455 // consider it only as a crate-level attribute.
456 if attr.has_name(sym::allow)
457 && let Some(list) = attr.meta_item_list()
458 && list.iter().any(|sub_attr| sub_attr.has_name(sym::internal_features))
459 {
460 push_to_s(&mut info.crate_attrs, source, attr.span, &mut prev_span_hi);
461 } else {
462 push_to_s(
463 &mut info.maybe_crate_attrs,
464 source,
465 attr.span,
466 &mut prev_span_hi,
467 );
468 }
469 } else {
470 push_to_s(&mut info.crate_attrs, source, attr.span, &mut prev_span_hi);
471 }
472 }
473 let mut has_non_items = false;
474 for stmt in &body.stmts {
475 let mut is_extern_crate = false;
476 match stmt.kind {
477 StmtKind::Item(ref item) => {
478 is_extern_crate = check_item(item, &mut info, crate_name);
479 }
480 // We assume that the macro calls will expand to item(s) even though they could
481 // expand to statements and expressions.
482 StmtKind::MacCall(ref mac_call) => {
483 if !info.has_main_fn {
484 // For backward compatibility, we look for the token sequence `fn main(…)`
485 // in the macro input (!) to crudely detect main functions "masked by a
486 // wrapper macro". For the record, this is a horrible heuristic!
487 // See <https://github.com/rust-lang/rust/issues/56898>.
488 let mut iter = mac_call.mac.args.tokens.iter();
489 while let Some(token) = iter.next() {
490 if let TokenTree::Token(token, _) = token
491 && let TokenKind::Ident(kw::Fn, _) = token.kind
492 && let Some(TokenTree::Token(ident, _)) = iter.peek()
493 && let TokenKind::Ident(sym::main, _) = ident.kind
494 && let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, _)) = {
495 iter.next();
496 iter.peek()
497 }
498 {
499 info.has_main_fn = true;
500 break;
501 }
502 }
503 }
504 }
505 StmtKind::Expr(ref expr) => {
506 if matches!(expr.kind, ast::ExprKind::Err(_)) {
507 reset_error_count(&psess);
508 return Err(());
509 }
510 has_non_items = true;
511 }
512 StmtKind::Let(_) | StmtKind::Semi(_) | StmtKind::Empty => has_non_items = true,
513 }
514
515 // Weirdly enough, the `Stmt` span doesn't include its attributes, so we need to
516 // tweak the span to include the attributes as well.
517 let mut span = stmt.span;
518 if let Some(attr) =
519 stmt.kind.attrs().iter().find(|attr| attr.style == AttrStyle::Outer)
520 {
521 span = span.with_lo(attr.span.lo());
522 }
523 if info.everything_else.is_empty()
524 && (!info.maybe_crate_attrs.is_empty() || !info.crate_attrs.is_empty())
525 {
526 // To keep the doctest code "as close as possible" to the original, we insert
527 // all the code located between this new span and the previous span which
528 // might contain code comments and backlines.
529 push_to_s(&mut info.crates, source, span.shrink_to_lo(), &mut prev_span_hi);
530 }
531 if !is_extern_crate {
532 push_to_s(&mut info.everything_else, source, span, &mut prev_span_hi);
533 } else {
534 push_to_s(&mut info.crates, source, span, &mut prev_span_hi);
535 }
536 }
537 if has_non_items {
538 if info.has_main_fn
539 && let Some(dcx) = parent_dcx
540 && !span.is_dummy()
541 {
542 dcx.span_warn(
543 span,
544 "the `main` function of this doctest won't be run as it contains \
545 expressions at the top level, meaning that the whole doctest code will be \
546 wrapped in a function",
547 );
548 }
549 info.has_main_fn = false;
550 }
551 Ok(info)
552 }
553 Err(e) => {
554 e.cancel();
555 Err(())
556 }
557 _ => Err(()),
558 };
559
560 reset_error_count(&psess);
561 result
562}