doctest.rs - source (original) (raw)
rustdoc/
doctest.rs
1mod extracted;
2mod make;
3mod markdown;
4mod runner;
5mod rust;
6
7use std::fs::File;
8use std::io::{self, Write};
9use std::path::{Path, PathBuf};
10use std::process::{self, Command, Stdio};
11use std::sync::atomic::{AtomicUsize, Ordering};
12use std::sync::{Arc, Mutex};
13use std::{panic, str};
14
15pub(crate) use make::DocTestBuilder;
16pub(crate) use markdown::test as test_markdown;
17use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
18use rustc_errors::emitter::HumanReadableErrorType;
19use rustc_errors::{ColorConfig, DiagCtxtHandle};
20use rustc_hir as hir;
21use rustc_hir::CRATE_HIR_ID;
22use rustc_hir::def_id::LOCAL_CRATE;
23use rustc_interface::interface;
24use rustc_session::config::{self, CrateType, ErrorOutputType, Input};
25use rustc_session::lint;
26use rustc_span::FileName;
27use rustc_span::edition::Edition;
28use rustc_span::symbol::sym;
29use rustc_target::spec::{Target, TargetTuple};
30use tempfile::{Builder as TempFileBuilder, TempDir};
31use tracing::debug;
32
33use self::rust::HirCollector;
34use crate::config::{Options as RustdocOptions, OutputFormat};
35use crate::html::markdown::{ErrorCodes, Ignore, LangString, MdRelLine};
36use crate::lint::init_lints;
37
38/// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`).
39#[derive(Clone)]
40pub(crate) struct GlobalTestOptions {
41 /// Name of the crate (for regular `rustdoc`) or Markdown file (for `rustdoc foo.md`).
42 pub(crate) crate_name: String,
43 /// Whether to disable the default `extern crate my_crate;` when creating doctests.
44 pub(crate) no_crate_inject: bool,
45 /// Whether inserting extra indent spaces in code block,
46 /// default is `false`, only `true` for generating code link of Rust playground
47 pub(crate) insert_indent_space: bool,
48 /// Additional crate-level attributes to add to doctests.
49 pub(crate) attrs: Vec<String>,
50 /// Path to file containing arguments for the invocation of rustc.
51 pub(crate) args_file: PathBuf,
52}
53
54pub(crate) fn generate_args_file(file_path: &Path, options: &RustdocOptions) -> Result<(), String> {
55 let mut file = File::create(file_path)
56 .map_err(|error| format!("failed to create args file: {error:?}"))?;
57
58 // We now put the common arguments into the file we created.
59 let mut content = vec![];
60
61 for cfg in &options.cfgs {
62 content.push(format!("--cfg={cfg}"));
63 }
64 for check_cfg in &options.check_cfgs {
65 content.push(format!("--check-cfg={check_cfg}"));
66 }
67
68 for lib_str in &options.lib_strs {
69 content.push(format!("-L{lib_str}"));
70 }
71 for extern_str in &options.extern_strs {
72 content.push(format!("--extern={extern_str}"));
73 }
74 content.push("-Ccodegen-units=1".to_string());
75 for codegen_options_str in &options.codegen_options_strs {
76 content.push(format!("-C{codegen_options_str}"));
77 }
78 for unstable_option_str in &options.unstable_opts_strs {
79 content.push(format!("-Z{unstable_option_str}"));
80 }
81
82 content.extend(options.doctest_build_args.clone());
83
84 let content = content.join("\n");
85
86 file.write_all(content.as_bytes())
87 .map_err(|error| format!("failed to write arguments to temporary file: {error:?}"))?;
88 Ok(())
89}
90
91fn get_doctest_dir() -> io::Result<TempDir> {
92 TempFileBuilder::new().prefix("rustdoctest").tempdir()
93}
94
95pub(crate) fn run(dcx: DiagCtxtHandle<'_>, input: Input, options: RustdocOptions) {
96 let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
97
98 // See core::create_config for what's going on here.
99 let allowed_lints = vec![
100 invalid_codeblock_attributes_name.to_owned(),
101 lint::builtin::UNKNOWN_LINTS.name.to_owned(),
102 lint::builtin::RENAMED_AND_REMOVED_LINTS.name.to_owned(),
103 ];
104
105 let (lint_opts, lint_caps) = init_lints(allowed_lints, options.lint_opts.clone(), |lint| {
106 if lint.name == invalid_codeblock_attributes_name {
107 None
108 } else {
109 Some((lint.name_lower(), lint::Allow))
110 }
111 });
112
113 debug!(?lint_opts);
114
115 let crate_types =
116 if options.proc_macro_crate { vec![CrateType::ProcMacro] } else { vec![CrateType::Rlib] };
117
118 let sessopts = config::Options {
119 sysroot: options.sysroot.clone(),
120 search_paths: options.libs.clone(),
121 crate_types,
122 lint_opts,
123 lint_cap: Some(options.lint_cap.unwrap_or(lint::Forbid)),
124 cg: options.codegen_options.clone(),
125 externs: options.externs.clone(),
126 unstable_features: options.unstable_features,
127 actually_rustdoc: true,
128 edition: options.edition,
129 target_triple: options.target.clone(),
130 crate_name: options.crate_name.clone(),
131 remap_path_prefix: options.remap_path_prefix.clone(),
132 ..config::Options::default()
133 };
134
135 let mut cfgs = options.cfgs.clone();
136 cfgs.push("doc".to_owned());
137 cfgs.push("doctest".to_owned());
138 let config = interface::Config {
139 opts: sessopts,
140 crate_cfg: cfgs,
141 crate_check_cfg: options.check_cfgs.clone(),
142 input: input.clone(),
143 output_file: None,
144 output_dir: None,
145 file_loader: None,
146 locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
147 lint_caps,
148 psess_created: None,
149 hash_untracked_state: None,
150 register_lints: Some(Box::new(crate::lint::register_lints)),
151 override_queries: None,
152 extra_symbols: Vec::new(),
153 make_codegen_backend: None,
154 registry: rustc_driver::diagnostics_registry(),
155 ice_file: None,
156 using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES,
157 expanded_args: options.expanded_args.clone(),
158 };
159
160 let externs = options.externs.clone();
161 let json_unused_externs = options.json_unused_externs;
162
163 let temp_dir = match get_doctest_dir()
164 .map_err(|error| format!("failed to create temporary directory: {error:?}"))
165 {
166 Ok(temp_dir) => temp_dir,
167 Err(error) => return crate::wrap_return(dcx, Err(error)),
168 };
169 let args_path = temp_dir.path().join("rustdoc-cfgs");
170 crate::wrap_return(dcx, generate_args_file(&args_path, &options));
171
172 let extract_doctests = options.output_format == OutputFormat::Doctest;
173 let result = interface::run_compiler(config, |compiler| {
174 let krate = rustc_interface::passes::parse(&compiler.sess);
175
176 let collector = rustc_interface::create_and_enter_global_ctxt(compiler, krate, |tcx| {
177 let crate_name = tcx.crate_name(LOCAL_CRATE).to_string();
178 let crate_attrs = tcx.hir_attrs(CRATE_HIR_ID);
179 let opts = scrape_test_config(crate_name, crate_attrs, args_path);
180
181 let hir_collector = HirCollector::new(
182 ErrorCodes::from(compiler.sess.opts.unstable_features.is_nightly_build()),
183 tcx,
184 );
185 let tests = hir_collector.collect_crate();
186 if extract_doctests {
187 let mut collector = extracted::ExtractedDocTests::new();
188 tests.into_iter().for_each(|t| collector.add_test(t, &opts, &options));
189
190 let stdout = std::io::stdout();
191 let mut stdout = stdout.lock();
192 if let Err(error) = serde_json::ser::to_writer(&mut stdout, &collector) {
193 eprintln!();
194 Err(format!("Failed to generate JSON output for doctests: {error:?}"))
195 } else {
196 Ok(None)
197 }
198 } else {
199 let mut collector = CreateRunnableDocTests::new(options, opts);
200 tests.into_iter().for_each(|t| collector.add_test(t));
201
202 Ok(Some(collector))
203 }
204 });
205 compiler.sess.dcx().abort_if_errors();
206
207 collector
208 });
209
210 let CreateRunnableDocTests {
211 standalone_tests,
212 mergeable_tests,
213 rustdoc_options,
214 opts,
215 unused_extern_reports,
216 compiling_test_count,
217 ..
218 } = match result {
219 Ok(Some(collector)) => collector,
220 Ok(None) => return,
221 Err(error) => {
222 eprintln!("{error}");
223 // Since some files in the temporary folder are still owned and alive, we need
224 // to manually remove the folder.
225 let _ = std::fs::remove_dir_all(temp_dir.path());
226 std::process::exit(1);
227 }
228 };
229
230 run_tests(
231 opts,
232 &rustdoc_options,
233 &unused_extern_reports,
234 standalone_tests,
235 mergeable_tests,
236 Some(temp_dir),
237 );
238
239 let compiling_test_count = compiling_test_count.load(Ordering::SeqCst);
240
241 // Collect and warn about unused externs, but only if we've gotten
242 // reports for each doctest
243 if json_unused_externs.is_enabled() {
244 let unused_extern_reports: Vec<_> =
245 std::mem::take(&mut unused_extern_reports.lock().unwrap());
246 if unused_extern_reports.len() == compiling_test_count {
247 let extern_names =
248 externs.iter().map(|(name, _)| name).collect::<FxIndexSet<&String>>();
249 let mut unused_extern_names = unused_extern_reports
250 .iter()
251 .map(|uexts| uexts.unused_extern_names.iter().collect::<FxIndexSet<&String>>())
252 .fold(extern_names, |uextsa, uextsb| {
253 uextsa.intersection(&uextsb).copied().collect::<FxIndexSet<&String>>()
254 })
255 .iter()
256 .map(|v| (*v).clone())
257 .collect::<Vec<String>>();
258 unused_extern_names.sort();
259 // Take the most severe lint level
260 let lint_level = unused_extern_reports
261 .iter()
262 .map(|uexts| uexts.lint_level.as_str())
263 .max_by_key(|v| match *v {
264 "warn" => 1,
265 "deny" => 2,
266 "forbid" => 3,
267 // The allow lint level is not expected,
268 // as if allow is specified, no message
269 // is to be emitted.
270 v => unreachable!("Invalid lint level '{v}'"),
271 })
272 .unwrap_or("warn")
273 .to_string();
274 let uext = UnusedExterns { lint_level, unused_extern_names };
275 let unused_extern_json = serde_json::to_string(&uext).unwrap();
276 eprintln!("{unused_extern_json}");
277 }
278 }
279}
280
281pub(crate) fn run_tests(
282 opts: GlobalTestOptions,
283 rustdoc_options: &Arc<RustdocOptions>,
284 unused_extern_reports: &Arc<Mutex<Vec<UnusedExterns>>>,
285 mut standalone_tests: Vec<test::TestDescAndFn>,
286 mergeable_tests: FxIndexMap<Edition, Vec<(DocTestBuilder, ScrapedDocTest)>>,
287 // We pass this argument so we can drop it manually before using `exit`.
288 mut temp_dir: Option<TempDir>,
289) {
290 let mut test_args = Vec::with_capacity(rustdoc_options.test_args.len() + 1);
291 test_args.insert(0, "rustdoctest".to_string());
292 test_args.extend_from_slice(&rustdoc_options.test_args);
293 if rustdoc_options.nocapture {
294 test_args.push("--nocapture".to_string());
295 }
296
297 let mut nb_errors = 0;
298 let mut ran_edition_tests = 0;
299 let target_str = rustdoc_options.target.to_string();
300
301 for (edition, mut doctests) in mergeable_tests {
302 if doctests.is_empty() {
303 continue;
304 }
305 doctests.sort_by(|(_, a), (_, b)| a.name.cmp(&b.name));
306
307 let mut tests_runner = runner::DocTestRunner::new();
308
309 let rustdoc_test_options = IndividualTestOptions::new(
310 rustdoc_options,
311 &Some(format!("merged_doctest_{edition}")),
312 PathBuf::from(format!("doctest_{edition}.rs")),
313 );
314
315 for (doctest, scraped_test) in &doctests {
316 tests_runner.add_test(doctest, scraped_test, &target_str);
317 }
318 if let Ok(success) = tests_runner.run_merged_tests(
319 rustdoc_test_options,
320 edition,
321 &opts,
322 &test_args,
323 rustdoc_options,
324 ) {
325 ran_edition_tests += 1;
326 if !success {
327 nb_errors += 1;
328 }
329 continue;
330 }
331 // We failed to compile all compatible tests as one so we push them into the
332 // `standalone_tests` doctests.
333 debug!("Failed to compile compatible doctests for edition {} all at once", edition);
334 for (doctest, scraped_test) in doctests {
335 doctest.generate_unique_doctest(
336 &scraped_test.text,
337 scraped_test.langstr.test_harness,
338 &opts,
339 Some(&opts.crate_name),
340 );
341 standalone_tests.push(generate_test_desc_and_fn(
342 doctest,
343 scraped_test,
344 opts.clone(),
345 Arc::clone(rustdoc_options),
346 unused_extern_reports.clone(),
347 ));
348 }
349 }
350
351 // We need to call `test_main` even if there is no doctest to run to get the output
352 // `running 0 tests...`.
353 if ran_edition_tests == 0 || !standalone_tests.is_empty() {
354 standalone_tests.sort_by(|a, b| a.desc.name.as_slice().cmp(b.desc.name.as_slice()));
355 test::test_main_with_exit_callback(&test_args, standalone_tests, None, || {
356 // We ensure temp dir destructor is called.
357 std::mem::drop(temp_dir.take());
358 });
359 }
360 if nb_errors != 0 {
361 // We ensure temp dir destructor is called.
362 std::mem::drop(temp_dir);
363 // libtest::ERROR_EXIT_CODE is not public but it's the same value.
364 std::process::exit(101);
365 }
366}
367
368// Look for `#![doc(test(no_crate_inject))]`, used by crates in the std facade.
369fn scrape_test_config(
370 crate_name: String,
371 attrs: &[hir::Attribute],
372 args_file: PathBuf,
373) -> GlobalTestOptions {
374 use rustc_ast_pretty::pprust;
375
376 let mut opts = GlobalTestOptions {
377 crate_name,
378 no_crate_inject: false,
379 attrs: Vec::new(),
380 insert_indent_space: false,
381 args_file,
382 };
383
384 let test_attrs: Vec<_> = attrs
385 .iter()
386 .filter(|a| a.has_name(sym::doc))
387 .flat_map(|a| a.meta_item_list().unwrap_or_default())
388 .filter(|a| a.has_name(sym::test))
389 .collect();
390 let attrs = test_attrs.iter().flat_map(|a| a.meta_item_list().unwrap_or(&[]));
391
392 for attr in attrs {
393 if attr.has_name(sym::no_crate_inject) {
394 opts.no_crate_inject = true;
395 }
396 if attr.has_name(sym::attr)
397 && let Some(l) = attr.meta_item_list()
398 {
399 for item in l {
400 opts.attrs.push(pprust::meta_list_item_to_string(item));
401 }
402 }
403 }
404
405 opts
406}
407
408/// Documentation test failure modes.
409enum TestFailure {
410 /// The test failed to compile.
411 CompileError,
412 /// The test is marked `compile_fail` but compiled successfully.
413 UnexpectedCompilePass,
414 /// The test failed to compile (as expected) but the compiler output did not contain all
415 /// expected error codes.
416 MissingErrorCodes(Vec<String>),
417 /// The test binary was unable to be executed.
418 ExecutionError(io::Error),
419 /// The test binary exited with a non-zero exit code.
420 ///
421 /// This typically means an assertion in the test failed or another form of panic occurred.
422 ExecutionFailure(process::Output),
423 /// The test is marked `should_panic` but the test binary executed successfully.
424 UnexpectedRunPass,
425}
426
427enum DirState {
428 Temp(TempDir),
429 Perm(PathBuf),
430}
431
432impl DirState {
433 fn path(&self) -> &std::path::Path {
434 match self {
435 DirState::Temp(t) => t.path(),
436 DirState::Perm(p) => p.as_path(),
437 }
438 }
439}
440
441// NOTE: Keep this in sync with the equivalent structs in rustc
442// and cargo.
443// We could unify this struct the one in rustc but they have different
444// ownership semantics, so doing so would create wasteful allocations.
445#[derive(serde::Serialize, serde::Deserialize)]
446pub(crate) struct UnusedExterns {
447 /// Lint level of the unused_crate_dependencies lint
448 lint_level: String,
449 /// List of unused externs by their names.
450 unused_extern_names: Vec<String>,
451}
452
453fn add_exe_suffix(input: String, target: &TargetTuple) -> String {
454 let exe_suffix = match target {
455 TargetTuple::TargetTuple(_) => Target::expect_builtin(target).options.exe_suffix,
456 TargetTuple::TargetJson { contents, .. } => {
457 Target::from_json(contents.parse().unwrap()).unwrap().0.options.exe_suffix
458 }
459 };
460 input + &exe_suffix
461}
462
463fn wrapped_rustc_command(rustc_wrappers: &[PathBuf], rustc_binary: &Path) -> Command {
464 let mut args = rustc_wrappers.iter().map(PathBuf::as_path).chain([rustc_binary]);
465
466 let exe = args.next().expect("unable to create rustc command");
467 let mut command = Command::new(exe);
468 for arg in args {
469 command.arg(arg);
470 }
471
472 command
473}
474
475/// Information needed for running a bundle of doctests.
476///
477/// This data structure contains the "full" test code, including the wrappers
478/// (if multiple doctests are merged), `main` function,
479/// and everything needed to calculate the compiler's command-line arguments.
480/// The `# ` prefix on boring lines has also been stripped.
481pub(crate) struct RunnableDocTest {
482 full_test_code: String,
483 full_test_line_offset: usize,
484 test_opts: IndividualTestOptions,
485 global_opts: GlobalTestOptions,
486 langstr: LangString,
487 line: usize,
488 edition: Edition,
489 no_run: bool,
490 merged_test_code: Option<String>,
491}
492
493impl RunnableDocTest {
494 fn path_for_merged_doctest_bundle(&self) -> PathBuf {
495 self.test_opts.outdir.path().join(format!("doctest_bundle_{}.rs", self.edition))
496 }
497 fn path_for_merged_doctest_runner(&self) -> PathBuf {
498 self.test_opts.outdir.path().join(format!("doctest_runner_{}.rs", self.edition))
499 }
500 fn is_multiple_tests(&self) -> bool {
501 self.merged_test_code.is_some()
502 }
503}
504
505/// Execute a `RunnableDoctest`.
506///
507/// This is the function that calculates the compiler command line, invokes the compiler, then
508/// invokes the test or tests in a separate executable (if applicable).
509fn run_test(
510 doctest: RunnableDocTest,
511 rustdoc_options: &RustdocOptions,
512 supports_color: bool,
513 report_unused_externs: impl Fn(UnusedExterns),
514) -> Result<(), TestFailure> {
515 let langstr = &doctest.langstr;
516 // Make sure we emit well-formed executable names for our target.
517 let rust_out = add_exe_suffix("rust_out".to_owned(), &rustdoc_options.target);
518 let output_file = doctest.test_opts.outdir.path().join(rust_out);
519
520 // Common arguments used for compiling the doctest runner.
521 // On merged doctests, the compiler is invoked twice: once for the test code itself,
522 // and once for the runner wrapper (which needs to use `#![feature]` on stable).
523 let mut compiler_args = vec![];
524
525 compiler_args.push(format!("@{}", doctest.global_opts.args_file.display()));
526
527 if let Some(sysroot) = &rustdoc_options.maybe_sysroot {
528 compiler_args.push(format!("--sysroot={}", sysroot.display()));
529 }
530
531 compiler_args.extend_from_slice(&["--edition".to_owned(), doctest.edition.to_string()]);
532 if langstr.test_harness {
533 compiler_args.push("--test".to_owned());
534 }
535 if rustdoc_options.json_unused_externs.is_enabled() && !langstr.compile_fail {
536 compiler_args.push("--error-format=json".to_owned());
537 compiler_args.extend_from_slice(&["--json".to_owned(), "unused-externs".to_owned()]);
538 compiler_args.extend_from_slice(&["-W".to_owned(), "unused_crate_dependencies".to_owned()]);
539 compiler_args.extend_from_slice(&["-Z".to_owned(), "unstable-options".to_owned()]);
540 }
541
542 if doctest.no_run && !langstr.compile_fail && rustdoc_options.persist_doctests.is_none() {
543 // FIXME: why does this code check if it *shouldn't* persist doctests
544 // -- shouldn't it be the negation?
545 compiler_args.push("--emit=metadata".to_owned());
546 }
547 compiler_args.extend_from_slice(&[
548 "--target".to_owned(),
549 match &rustdoc_options.target {
550 TargetTuple::TargetTuple(s) => s.clone(),
551 TargetTuple::TargetJson { path_for_rustdoc, .. } => {
552 path_for_rustdoc.to_str().expect("target path must be valid unicode").to_owned()
553 }
554 },
555 ]);
556 if let ErrorOutputType::HumanReadable { kind, color_config } = rustdoc_options.error_format {
557 let short = kind.short();
558 let unicode = kind == HumanReadableErrorType::Unicode;
559
560 if short {
561 compiler_args.extend_from_slice(&["--error-format".to_owned(), "short".to_owned()]);
562 }
563 if unicode {
564 compiler_args
565 .extend_from_slice(&["--error-format".to_owned(), "human-unicode".to_owned()]);
566 }
567
568 match color_config {
569 ColorConfig::Never => {
570 compiler_args.extend_from_slice(&["--color".to_owned(), "never".to_owned()]);
571 }
572 ColorConfig::Always => {
573 compiler_args.extend_from_slice(&["--color".to_owned(), "always".to_owned()]);
574 }
575 ColorConfig::Auto => {
576 compiler_args.extend_from_slice(&[
577 "--color".to_owned(),
578 if supports_color { "always" } else { "never" }.to_owned(),
579 ]);
580 }
581 }
582 }
583
584 let rustc_binary = rustdoc_options
585 .test_builder
586 .as_deref()
587 .unwrap_or_else(|| rustc_interface::util::rustc_path().expect("found rustc"));
588 let mut compiler = wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
589
590 compiler.args(&compiler_args);
591
592 // If this is a merged doctest, we need to write it into a file instead of using stdin
593 // because if the size of the merged doctests is too big, it'll simply break stdin.
594 if doctest.is_multiple_tests() {
595 // It makes the compilation failure much faster if it is for a combined doctest.
596 compiler.arg("--error-format=short");
597 let input_file = doctest.path_for_merged_doctest_bundle();
598 if std::fs::write(&input_file, &doctest.full_test_code).is_err() {
599 // If we cannot write this file for any reason, we leave. All combined tests will be
600 // tested as standalone tests.
601 return Err(TestFailure::CompileError);
602 }
603 if !rustdoc_options.nocapture {
604 // If `nocapture` is disabled, then we don't display rustc's output when compiling
605 // the merged doctests.
606 compiler.stderr(Stdio::null());
607 }
608 // bundled tests are an rlib, loaded by a separate runner executable
609 compiler
610 .arg("--crate-type=lib")
611 .arg("--out-dir")
612 .arg(doctest.test_opts.outdir.path())
613 .arg(input_file);
614 } else {
615 compiler.arg("--crate-type=bin").arg("-o").arg(&output_file);
616 // Setting these environment variables is unneeded if this is a merged doctest.
617 compiler.env("UNSTABLE_RUSTDOC_TEST_PATH", &doctest.test_opts.path);
618 compiler.env(
619 "UNSTABLE_RUSTDOC_TEST_LINE",
620 format!("{}", doctest.line as isize - doctest.full_test_line_offset as isize),
621 );
622 compiler.arg("-");
623 compiler.stdin(Stdio::piped());
624 compiler.stderr(Stdio::piped());
625 }
626
627 debug!("compiler invocation for doctest: {compiler:?}");
628
629 let mut child = compiler.spawn().expect("Failed to spawn rustc process");
630 let output = if let Some(merged_test_code) = &doctest.merged_test_code {
631 // compile-fail tests never get merged, so this should always pass
632 let status = child.wait().expect("Failed to wait");
633
634 // the actual test runner is a separate component, built with nightly-only features;
635 // build it now
636 let runner_input_file = doctest.path_for_merged_doctest_runner();
637
638 let mut runner_compiler =
639 wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
640 // the test runner does not contain any user-written code, so this doesn't allow
641 // the user to exploit nightly-only features on stable
642 runner_compiler.env("RUSTC_BOOTSTRAP", "1");
643 runner_compiler.args(compiler_args);
644 runner_compiler.args(&["--crate-type=bin", "-o"]).arg(&output_file);
645 let mut extern_path = std::ffi::OsString::from(format!(
646 "--extern=doctest_bundle_{edition}=",
647 edition = doctest.edition
648 ));
649 for extern_str in &rustdoc_options.extern_strs {
650 if let Some((_cratename, path)) = extern_str.split_once('=') {
651 // Direct dependencies of the tests themselves are
652 // indirect dependencies of the test runner.
653 // They need to be in the library search path.
654 let dir = Path::new(path)
655 .parent()
656 .filter(|x| x.components().count() > 0)
657 .unwrap_or(Path::new("."));
658 runner_compiler.arg("-L").arg(dir);
659 }
660 }
661 let output_bundle_file = doctest
662 .test_opts
663 .outdir
664 .path()
665 .join(format!("libdoctest_bundle_{edition}.rlib", edition = doctest.edition));
666 extern_path.push(&output_bundle_file);
667 runner_compiler.arg(extern_path);
668 runner_compiler.arg(&runner_input_file);
669 if std::fs::write(&runner_input_file, &merged_test_code).is_err() {
670 // If we cannot write this file for any reason, we leave. All combined tests will be
671 // tested as standalone tests.
672 return Err(TestFailure::CompileError);
673 }
674 if !rustdoc_options.nocapture {
675 // If `nocapture` is disabled, then we don't display rustc's output when compiling
676 // the merged doctests.
677 runner_compiler.stderr(Stdio::null());
678 }
679 runner_compiler.arg("--error-format=short");
680 debug!("compiler invocation for doctest runner: {runner_compiler:?}");
681
682 let status = if !status.success() {
683 status
684 } else {
685 let mut child_runner = runner_compiler.spawn().expect("Failed to spawn rustc process");
686 child_runner.wait().expect("Failed to wait")
687 };
688
689 process::Output { status, stdout: Vec::new(), stderr: Vec::new() }
690 } else {
691 let stdin = child.stdin.as_mut().expect("Failed to open stdin");
692 stdin.write_all(doctest.full_test_code.as_bytes()).expect("could write out test sources");
693 child.wait_with_output().expect("Failed to read stdout")
694 };
695
696 struct Bomb<'a>(&'a str);
697 impl Drop for Bomb<'_> {
698 fn drop(&mut self) {
699 eprint!("{}", self.0);
700 }
701 }
702 let mut out = str::from_utf8(&output.stderr)
703 .unwrap()
704 .lines()
705 .filter(|l| {
706 if let Ok(uext) = serde_json::from_str::<UnusedExterns>(l) {
707 report_unused_externs(uext);
708 false
709 } else {
710 true
711 }
712 })
713 .intersperse_with(|| "\n")
714 .collect::<String>();
715
716 // Add a \n to the end to properly terminate the last line,
717 // but only if there was output to be printed
718 if !out.is_empty() {
719 out.push('\n');
720 }
721
722 let _bomb = Bomb(&out);
723 match (output.status.success(), langstr.compile_fail) {
724 (true, true) => {
725 return Err(TestFailure::UnexpectedCompilePass);
726 }
727 (true, false) => {}
728 (false, true) => {
729 if !langstr.error_codes.is_empty() {
730 // We used to check if the output contained "error[{}]: " but since we added the
731 // colored output, we can't anymore because of the color escape characters before
732 // the ":".
733 let missing_codes: Vec<String> = langstr
734 .error_codes
735 .iter()
736 .filter(|err| !out.contains(&format!("error[{err}]")))
737 .cloned()
738 .collect();
739
740 if !missing_codes.is_empty() {
741 return Err(TestFailure::MissingErrorCodes(missing_codes));
742 }
743 }
744 }
745 (false, false) => {
746 return Err(TestFailure::CompileError);
747 }
748 }
749
750 if doctest.no_run {
751 return Ok(());
752 }
753
754 // Run the code!
755 let mut cmd;
756
757 let output_file = make_maybe_absolute_path(output_file);
758 if let Some(tool) = &rustdoc_options.test_runtool {
759 let tool = make_maybe_absolute_path(tool.into());
760 cmd = Command::new(tool);
761 cmd.args(&rustdoc_options.test_runtool_args);
762 cmd.arg(&output_file);
763 } else {
764 cmd = Command::new(&output_file);
765 if doctest.is_multiple_tests() {
766 cmd.env("RUSTDOC_DOCTEST_BIN_PATH", &output_file);
767 }
768 }
769 if let Some(run_directory) = &rustdoc_options.test_run_directory {
770 cmd.current_dir(run_directory);
771 }
772
773 let result = if doctest.is_multiple_tests() || rustdoc_options.nocapture {
774 cmd.status().map(|status| process::Output {
775 status,
776 stdout: Vec::new(),
777 stderr: Vec::new(),
778 })
779 } else {
780 cmd.output()
781 };
782 match result {
783 Err(e) => return Err(TestFailure::ExecutionError(e)),
784 Ok(out) => {
785 if langstr.should_panic && out.status.success() {
786 return Err(TestFailure::UnexpectedRunPass);
787 } else if !langstr.should_panic && !out.status.success() {
788 return Err(TestFailure::ExecutionFailure(out));
789 }
790 }
791 }
792
793 Ok(())
794}
795
796/// Converts a path intended to use as a command to absolute if it is
797/// relative, and not a single component.
798///
799/// This is needed to deal with relative paths interacting with
800/// `Command::current_dir` in a platform-specific way.
801fn make_maybe_absolute_path(path: PathBuf) -> PathBuf {
802 if path.components().count() == 1 {
803 // Look up process via PATH.
804 path
805 } else {
806 std::env::current_dir().map(|c| c.join(&path)).unwrap_or_else(|_| path)
807 }
808}
809struct IndividualTestOptions {
810 outdir: DirState,
811 path: PathBuf,
812}
813
814impl IndividualTestOptions {
815 fn new(options: &RustdocOptions, test_id: &Option<String>, test_path: PathBuf) -> Self {
816 let outdir = if let Some(ref path) = options.persist_doctests {
817 let mut path = path.clone();
818 path.push(test_id.as_deref().unwrap_or("<doctest>"));
819
820 if let Err(err) = std::fs::create_dir_all(&path) {
821 eprintln!("Couldn't create directory for doctest executables: {err}");
822 panic::resume_unwind(Box::new(()));
823 }
824
825 DirState::Perm(path)
826 } else {
827 DirState::Temp(get_doctest_dir().expect("rustdoc needs a tempdir"))
828 };
829
830 Self { outdir, path: test_path }
831 }
832}
833
834/// A doctest scraped from the code, ready to be turned into a runnable test.
835///
836/// The pipeline goes: [`clean`] AST -> `ScrapedDoctest` -> `RunnableDoctest`.
837/// [`run_merged_tests`] converts a bunch of scraped doctests to a single runnable doctest,
838/// while [`generate_unique_doctest`] does the standalones.
839///
840/// [`clean`]: crate::clean
841/// [`run_merged_tests`]: crate::doctest::runner::DocTestRunner::run_merged_tests
842/// [`generate_unique_doctest`]: crate::doctest::make::DocTestBuilder::generate_unique_doctest
843#[derive(Debug)]
844pub(crate) struct ScrapedDocTest {
845 filename: FileName,
846 line: usize,
847 langstr: LangString,
848 text: String,
849 name: String,
850}
851
852impl ScrapedDocTest {
853 fn new(
854 filename: FileName,
855 line: usize,
856 logical_path: Vec<String>,
857 langstr: LangString,
858 text: String,
859 ) -> Self {
860 let mut item_path = logical_path.join("::");
861 item_path.retain(|c| c != ' ');
862 if !item_path.is_empty() {
863 item_path.push(' ');
864 }
865 let name =
866 format!("{} - {item_path}(line {line})", filename.prefer_remapped_unconditionaly());
867
868 Self { filename, line, langstr, text, name }
869 }
870 fn edition(&self, opts: &RustdocOptions) -> Edition {
871 self.langstr.edition.unwrap_or(opts.edition)
872 }
873
874 fn no_run(&self, opts: &RustdocOptions) -> bool {
875 self.langstr.no_run || opts.no_run
876 }
877 fn path(&self) -> PathBuf {
878 match &self.filename {
879 FileName::Real(path) => {
880 if let Some(local_path) = path.local_path() {
881 local_path.to_path_buf()
882 } else {
883 // Somehow we got the filename from the metadata of another crate, should never happen
884 unreachable!("doctest from a different crate");
885 }
886 }
887 _ => PathBuf::from(r"doctest.rs"),
888 }
889 }
890}
891
892pub(crate) trait DocTestVisitor {
893 fn visit_test(&mut self, test: String, config: LangString, rel_line: MdRelLine);
894 fn visit_header(&mut self, _name: &str, _level: u32) {}
895}
896
897struct CreateRunnableDocTests {
898 standalone_tests: Vec<test::TestDescAndFn>,
899 mergeable_tests: FxIndexMap<Edition, Vec<(DocTestBuilder, ScrapedDocTest)>>,
900
901 rustdoc_options: Arc<RustdocOptions>,
902 opts: GlobalTestOptions,
903 visited_tests: FxHashMap<(String, usize), usize>,
904 unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>,
905 compiling_test_count: AtomicUsize,
906 can_merge_doctests: bool,
907}
908
909impl CreateRunnableDocTests {
910 fn new(rustdoc_options: RustdocOptions, opts: GlobalTestOptions) -> CreateRunnableDocTests {
911 let can_merge_doctests = rustdoc_options.edition >= Edition::Edition2024;
912 CreateRunnableDocTests {
913 standalone_tests: Vec::new(),
914 mergeable_tests: FxIndexMap::default(),
915 rustdoc_options: Arc::new(rustdoc_options),
916 opts,
917 visited_tests: FxHashMap::default(),
918 unused_extern_reports: Default::default(),
919 compiling_test_count: AtomicUsize::new(0),
920 can_merge_doctests,
921 }
922 }
923
924 fn add_test(&mut self, scraped_test: ScrapedDocTest) {
925 // For example `module/file.rs` would become `module_file_rs`
926 let file = scraped_test
927 .filename
928 .prefer_local()
929 .to_string_lossy()
930 .chars()
931 .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
932 .collect::<String>();
933 let test_id = format!(
934 "{file}_{line}_{number}",
935 file = file,
936 line = scraped_test.line,
937 number = {
938 // Increases the current test number, if this file already
939 // exists or it creates a new entry with a test number of 0.
940 self.visited_tests
941 .entry((file.clone(), scraped_test.line))
942 .and_modify(|v| *v += 1)
943 .or_insert(0)
944 },
945 );
946
947 let edition = scraped_test.edition(&self.rustdoc_options);
948 let doctest = DocTestBuilder::new(
949 &scraped_test.text,
950 Some(&self.opts.crate_name),
951 edition,
952 self.can_merge_doctests,
953 Some(test_id),
954 Some(&scraped_test.langstr),
955 );
956 let is_standalone = !doctest.can_be_merged
957 || scraped_test.langstr.compile_fail
958 || scraped_test.langstr.test_harness
959 || scraped_test.langstr.standalone_crate
960 || self.rustdoc_options.nocapture
961 || self.rustdoc_options.test_args.iter().any(|arg| arg == "--show-output");
962 if is_standalone {
963 let test_desc = self.generate_test_desc_and_fn(doctest, scraped_test);
964 self.standalone_tests.push(test_desc);
965 } else {
966 self.mergeable_tests.entry(edition).or_default().push((doctest, scraped_test));
967 }
968 }
969
970 fn generate_test_desc_and_fn(
971 &mut self,
972 test: DocTestBuilder,
973 scraped_test: ScrapedDocTest,
974 ) -> test::TestDescAndFn {
975 if !scraped_test.langstr.compile_fail {
976 self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
977 }
978
979 generate_test_desc_and_fn(
980 test,
981 scraped_test,
982 self.opts.clone(),
983 Arc::clone(&self.rustdoc_options),
984 self.unused_extern_reports.clone(),
985 )
986 }
987}
988
989fn generate_test_desc_and_fn(
990 test: DocTestBuilder,
991 scraped_test: ScrapedDocTest,
992 opts: GlobalTestOptions,
993 rustdoc_options: Arc<RustdocOptions>,
994 unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
995) -> test::TestDescAndFn {
996 let target_str = rustdoc_options.target.to_string();
997 let rustdoc_test_options =
998 IndividualTestOptions::new(&rustdoc_options, &test.test_id, scraped_test.path());
999
1000 debug!("creating test {}: {}", scraped_test.name, scraped_test.text);
1001 test::TestDescAndFn {
1002 desc: test::TestDesc {
1003 name: test::DynTestName(scraped_test.name.clone()),
1004 ignore: match scraped_test.langstr.ignore {
1005 Ignore::All => true,
1006 Ignore::None => false,
1007 Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
1008 },
1009 ignore_message: None,
1010 source_file: "",
1011 start_line: 0,
1012 start_col: 0,
1013 end_line: 0,
1014 end_col: 0,
1015 // compiler failures are test failures
1016 should_panic: test::ShouldPanic::No,
1017 compile_fail: scraped_test.langstr.compile_fail,
1018 no_run: scraped_test.no_run(&rustdoc_options),
1019 test_type: test::TestType::DocTest,
1020 },
1021 testfn: test::DynTestFn(Box::new(move || {
1022 doctest_run_fn(
1023 rustdoc_test_options,
1024 opts,
1025 test,
1026 scraped_test,
1027 rustdoc_options,
1028 unused_externs,
1029 )
1030 })),
1031 }
1032}
1033
1034fn doctest_run_fn(
1035 test_opts: IndividualTestOptions,
1036 global_opts: GlobalTestOptions,
1037 doctest: DocTestBuilder,
1038 scraped_test: ScrapedDocTest,
1039 rustdoc_options: Arc<RustdocOptions>,
1040 unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
1041) -> Result<(), String> {
1042 let report_unused_externs = |uext| {
1043 unused_externs.lock().unwrap().push(uext);
1044 };
1045 let (full_test_code, full_test_line_offset) = doctest.generate_unique_doctest(
1046 &scraped_test.text,
1047 scraped_test.langstr.test_harness,
1048 &global_opts,
1049 Some(&global_opts.crate_name),
1050 );
1051 let runnable_test = RunnableDocTest {
1052 full_test_code,
1053 full_test_line_offset,
1054 test_opts,
1055 global_opts,
1056 langstr: scraped_test.langstr.clone(),
1057 line: scraped_test.line,
1058 edition: scraped_test.edition(&rustdoc_options),
1059 no_run: scraped_test.no_run(&rustdoc_options),
1060 merged_test_code: None,
1061 };
1062 let res =
1063 run_test(runnable_test, &rustdoc_options, doctest.supports_color, report_unused_externs);
1064
1065 if let Err(err) = res {
1066 match err {
1067 TestFailure::CompileError => {
1068 eprint!("Couldn't compile the test.");
1069 }
1070 TestFailure::UnexpectedCompilePass => {
1071 eprint!("Test compiled successfully, but it's marked `compile_fail`.");
1072 }
1073 TestFailure::UnexpectedRunPass => {
1074 eprint!("Test executable succeeded, but it's marked `should_panic`.");
1075 }
1076 TestFailure::MissingErrorCodes(codes) => {
1077 eprint!("Some expected error codes were not found: {codes:?}");
1078 }
1079 TestFailure::ExecutionError(err) => {
1080 eprint!("Couldn't run the test: {err}");
1081 if err.kind() == io::ErrorKind::PermissionDenied {
1082 eprint!(" - maybe your tempdir is mounted with noexec?");
1083 }
1084 }
1085 TestFailure::ExecutionFailure(out) => {
1086 eprintln!("Test executable failed ({reason}).", reason = out.status);
1087
1088 // FIXME(#12309): An unfortunate side-effect of capturing the test
1089 // executable's output is that the relative ordering between the test's
1090 // stdout and stderr is lost. However, this is better than the
1091 // alternative: if the test executable inherited the parent's I/O
1092 // handles the output wouldn't be captured at all, even on success.
1093 //
1094 // The ordering could be preserved if the test process' stderr was
1095 // redirected to stdout, but that functionality does not exist in the
1096 // standard library, so it may not be portable enough.
1097 let stdout = str::from_utf8(&out.stdout).unwrap_or_default();
1098 let stderr = str::from_utf8(&out.stderr).unwrap_or_default();
1099
1100 if !stdout.is_empty() || !stderr.is_empty() {
1101 eprintln!();
1102
1103 if !stdout.is_empty() {
1104 eprintln!("stdout:\n{stdout}");
1105 }
1106
1107 if !stderr.is_empty() {
1108 eprintln!("stderr:\n{stderr}");
1109 }
1110 }
1111 }
1112 }
1113
1114 panic::resume_unwind(Box::new(()));
1115 }
1116 Ok(())
1117}
1118
1119#[cfg(test)] // used in tests
1120impl DocTestVisitor for Vec<usize> {
1121 fn visit_test(&mut self, _test: String, _config: LangString, rel_line: MdRelLine) {
1122 self.push(1 + rel_line.offset());
1123 }
1124}
1125
1126#[cfg(test)]
1127mod tests;