Squashed 'third_party/autocxx/' content from commit 629e8fa53
git-subtree-dir: third_party/autocxx
git-subtree-split: 629e8fa531a633164c0b52e2a3cab536d4cd0849
Signed-off-by: Brian Silverman <bsilver16384@gmail.com>
Change-Id: I62a03b0049f49adf029e0204639cdb5468dde1a1
diff --git a/tools/reduce/src/main.rs b/tools/reduce/src/main.rs
new file mode 100644
index 0000000..17c5bf0
--- /dev/null
+++ b/tools/reduce/src/main.rs
@@ -0,0 +1,562 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![forbid(unsafe_code)]
+
+use std::{
+ borrow::Cow,
+ fs::File,
+ io::Write,
+ os::unix::prelude::PermissionsExt,
+ path::{Path, PathBuf},
+};
+
+use autocxx_engine::{get_clang_path, make_clang_args, preprocess};
+use autocxx_parser::IncludeCppConfig;
+use clap::{crate_authors, crate_version, Arg, ArgMatches, Command};
+use indexmap::IndexSet;
+use indoc::indoc;
+use itertools::Itertools;
+use quote::ToTokens;
+use regex::Regex;
+use tempfile::TempDir;
+
+static LONG_HELP: &str = indoc! {"
+Command line utility to minimize autocxx bug cases.
+
+This is a wrapper for creduce.
+
+Example command-line:
+autocxx-reduce file -I my-inc-dir -h my-header -d 'generate!(\"MyClass\")' -k -- --n 64
+"};
+
+fn main() {
+ let matches = Command::new("autocxx-reduce")
+ .version(crate_version!())
+ .author(crate_authors!())
+ .about("Reduce a C++ test case")
+ .long_about(LONG_HELP)
+ .subcommand(Command::new("file")
+ .about("reduce a header file")
+
+ .arg(
+ Arg::new("inc")
+ .short('I')
+ .long("inc")
+ .multiple_occurrences(true)
+ .number_of_values(1)
+ .value_name("INCLUDE DIRS")
+ .help("include path")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("define")
+ .short('D')
+ .long("define")
+ .multiple_occurrences(true)
+ .number_of_values(1)
+ .value_name("DEFINE")
+ .help("macro definition")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("header")
+ .long("header")
+ .multiple_occurrences(true)
+ .number_of_values(1)
+ .required(true)
+ .value_name("HEADER")
+ .help("header file name")
+ .takes_value(true),
+ )
+
+ .arg(
+ Arg::new("directive")
+ .short('d')
+ .long("directive")
+ .multiple_occurrences(true)
+ .number_of_values(1)
+ .value_name("DIRECTIVE")
+ .help("directives to put within include_cpp!")
+ .takes_value(true),
+ )
+ )
+ .subcommand(Command::new("repro")
+ .about("reduce a repro case JSON file")
+ .arg(
+ Arg::new("repro")
+ .short('r')
+ .long("repro")
+ .required(true)
+ .value_name("REPRODUCTION CASE JSON")
+ .help("reproduction case JSON file name")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("header")
+ .long("header")
+ .multiple_occurrences(true)
+ .number_of_values(1)
+ .value_name("HEADER")
+ .help("header file name; specify to resume a part-completed run")
+ .takes_value(true),
+ )
+ )
+ .arg(
+ Arg::new("problem")
+ .short('p')
+ .long("problem")
+ .required(true)
+ .value_name("PROBLEM")
+ .help("problem string we're looking for... may be in logs, or in generated C++, or generated .rs")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("creduce")
+ .long("creduce")
+ .value_name("PATH")
+ .help("creduce binary location")
+ .default_value("creduce")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("output")
+ .short('o')
+ .long("output")
+ .value_name("OUTPUT")
+ .help("where to write minimized output")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("gen-cmd")
+ .short('g')
+ .long("gen-cmd")
+ .value_name("GEN-CMD")
+ .help("where to find autocxx-gen")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("keep")
+ .short('k')
+ .long("keep-dir")
+ .help("keep the temporary directory for debugging purposes"),
+ )
+ .arg(
+ Arg::new("clang-args")
+ .short('c')
+ .long("clang-arg")
+ .multiple_occurrences(true)
+ .value_name("CLANG_ARG")
+ .help("Extra arguments to pass to Clang"),
+ )
+ .arg(
+ Arg::new("creduce-args")
+ .long("creduce-arg")
+ .multiple_occurrences(true)
+ .value_name("CREDUCE_ARG")
+ .help("Extra arguments to pass to Clang"),
+ )
+ .arg(
+ Arg::new("no-precompile")
+ .long("no-precompile")
+ .help("Do not precompile the C++ header before passing to autocxxgen"),
+ )
+ .arg(
+ Arg::new("no-postcompile")
+ .long("no-postcompile")
+ .help("Do not post-compile the C++ generated by autocxxgen"),
+ )
+ .arg(
+ Arg::new("suppress-cxx-inclusions")
+ .long("suppress-cxx-inclusions")
+ .takes_value(true)
+ .possible_value("yes")
+ .possible_value("no")
+ .possible_value("auto")
+ .default_value("auto")
+ .help("Whether the preprocessed header already includes cxx.h. If so, we'll try to suppress the natural behavior of cxx to include duplicate definitions of some of the types within gen0.cc.")
+ )
+ .arg_required_else_help(true)
+ .get_matches();
+ run(matches).unwrap();
+}
+
+fn run(matches: ArgMatches) -> Result<(), std::io::Error> {
+ let keep_tmp = matches.is_present("keep");
+ let tmp_dir = TempDir::new()?;
+ let r = do_run(matches, &tmp_dir);
+ if keep_tmp {
+ println!(
+ "Keeping temp dir created at: {}",
+ tmp_dir.into_path().to_str().unwrap()
+ );
+ }
+ r
+}
+
+#[derive(serde_derive::Deserialize)]
+struct ReproCase {
+ config: String,
+ header: String,
+}
+
+fn do_run(matches: ArgMatches, tmp_dir: &TempDir) -> Result<(), std::io::Error> {
+ let rs_path = tmp_dir.path().join("input.rs");
+ let concat_path = tmp_dir.path().join("concat.h");
+ match matches.subcommand_matches("repro") {
+ None => {
+ let submatches = matches.subcommand_matches("file").unwrap();
+ let incs: Vec<_> = submatches
+ .values_of("inc")
+ .unwrap_or_default()
+ .map(PathBuf::from)
+ .collect();
+ let defs: Vec<_> = submatches.values_of("define").unwrap_or_default().collect();
+ let headers: Vec<_> = submatches.values_of("header").unwrap_or_default().collect();
+ assert!(!headers.is_empty());
+ let listing_path = tmp_dir.path().join("listing.h");
+ create_concatenated_header(&headers, &listing_path)?;
+ announce_progress(&format!(
+ "Preprocessing {:?} to {:?}",
+ listing_path, concat_path
+ ));
+ preprocess(&listing_path, &concat_path, &incs, &defs)?;
+ let directives: Vec<_> = std::iter::once("#include \"concat.h\"\n".to_string())
+ .chain(
+ submatches
+ .values_of("directive")
+ .unwrap_or_default()
+ .map(|s| format!("{}\n", s)),
+ )
+ .collect();
+ create_rs_file(&rs_path, &directives)?;
+ }
+ Some(submatches) => {
+ let case: ReproCase = serde_json::from_reader(File::open(PathBuf::from(
+ submatches.value_of("repro").unwrap(),
+ ))?)
+ .unwrap();
+ // Replace the headers in the config
+ let mut config: IncludeCppConfig = syn::parse_str(&case.config).unwrap();
+ config.replace_included_headers("concat.h");
+ create_file(
+ &rs_path,
+ &format!("autocxx::include_cpp!({});", config.to_token_stream()),
+ )?;
+ if let Some(header) = submatches.value_of("header") {
+ std::fs::copy(PathBuf::from(header), &concat_path)?;
+ } else {
+ create_file(&concat_path, &case.header)?
+ }
+ }
+ }
+
+ let suppress_cxx_classes = match matches.value_of("suppress-cxx-inclusions").unwrap() {
+ "yes" => true,
+ "no" => false,
+ "auto" => detect_cxx_h(&concat_path)?,
+ _ => panic!("unexpected value"),
+ };
+
+ let cxx_suppressions = if suppress_cxx_classes {
+ get_cxx_suppressions()
+ } else {
+ Vec::new()
+ };
+
+ let extra_clang_args: Vec<_> = matches
+ .values_of("clang-args")
+ .unwrap_or_default()
+ .map(Cow::Borrowed)
+ .chain(cxx_suppressions.into_iter().map(Cow::Owned))
+ .collect();
+ let extra_clang_args: Vec<&str> = extra_clang_args.iter().map(|s| s.as_ref()).collect_vec();
+
+ let default_gen_cmd = std::env::current_exe()?
+ .parent()
+ .unwrap()
+ .join("autocxx-gen")
+ .to_str()
+ .unwrap()
+ .to_string();
+ let gen_cmd = matches.value_of("gen-cmd").unwrap_or(&default_gen_cmd);
+ if !Path::new(gen_cmd).exists() {
+ panic!(
+ "autocxx-gen not found in {}. hint: autocxx-reduce --gen-cmd /path/to/autocxx-gen",
+ gen_cmd
+ );
+ }
+ run_sample_gen_cmd(gen_cmd, &rs_path, tmp_dir.path(), &extra_clang_args)?;
+ // Create and run an interestingness test which does not filter its output through grep.
+ let demo_interestingness_test_dir = tmp_dir.path().join("demo-interestingness-test");
+ std::fs::create_dir(&demo_interestingness_test_dir).unwrap();
+ let interestingness_test = demo_interestingness_test_dir.join("test-demo.sh");
+ create_interestingness_test(
+ gen_cmd,
+ &interestingness_test,
+ None,
+ &rs_path,
+ &extra_clang_args,
+ !matches.is_present("no-precompile"),
+ !matches.is_present("no-postcompile"),
+ )?;
+ let demo_dir_concat_path = demo_interestingness_test_dir.join("concat.h");
+ std::fs::copy(&concat_path, demo_dir_concat_path).unwrap();
+ run_demo_interestingness_test(&demo_interestingness_test_dir, &interestingness_test).unwrap();
+
+ // Now the main interestingness test
+ let interestingness_test = tmp_dir.path().join("test.sh");
+ create_interestingness_test(
+ gen_cmd,
+ &interestingness_test,
+ Some(matches.value_of("problem").unwrap()),
+ &rs_path,
+ &extra_clang_args,
+ !matches.is_present("no-precompile"),
+ !matches.is_present("no-postcompile"),
+ )?;
+ run_creduce(
+ matches.value_of("creduce").unwrap(),
+ &interestingness_test,
+ &concat_path,
+ matches.values_of("creduce-args").unwrap_or_default(),
+ );
+ announce_progress("creduce completed");
+ let output_path = matches.value_of("output");
+ match output_path {
+ None => print_minimized_case(&concat_path)?,
+ Some(output_path) => {
+ std::fs::copy(&concat_path, &PathBuf::from(output_path))?;
+ }
+ };
+ Ok(())
+}
+
+/// Try to detect whether the preprocessed source code already contains
+/// a preprocessed version of cxx.h. This is hard because all the comments
+/// and preprocessor symbols may have been removed, and in fact if we're
+/// part way through reduction, parts of the code may have been removed too.
+fn detect_cxx_h(concat_path: &Path) -> Result<bool, std::io::Error> {
+ let haystack = std::fs::read_to_string(concat_path)?;
+ Ok(["class Box", "class Vec", "class Slice"]
+ .iter()
+ .all(|needle| haystack.contains(needle)))
+}
+
+fn announce_progress(msg: &str) {
+ println!("=== {} ===", msg);
+}
+
+fn print_minimized_case(concat_path: &Path) -> Result<(), std::io::Error> {
+ announce_progress("Completed. Minimized test case:");
+ let contents = std::fs::read_to_string(concat_path)?;
+ println!("{}", contents);
+ Ok(())
+}
+
+/// Arguments we pass to creduce if supported. This pass always seems to cause a crash
+/// as far as I can tell, so always exclude it. It may be environment-dependent,
+/// of course, but as I'm the primary user of this tool I am ruthlessly removing it.
+const REMOVE_PASS_LINE_MARKERS: &[&str] = &["--remove-pass", "pass_line_markers", "*"];
+const SKIP_INITIAL_PASSES: &[&str] = &["--skip-initial-passes"];
+
+fn creduce_supports_remove_pass(creduce_cmd: &str) -> bool {
+ let cmd = std::process::Command::new(creduce_cmd)
+ .arg("--help")
+ .output();
+ let msg = match cmd {
+ Err(error) => panic!("failed to run creduce. creduce_cmd = {}. hint: autocxx-reduce --creduce /path/to/creduce. error = {}", creduce_cmd, error),
+ Ok(result) => result.stdout
+ };
+ let msg = std::str::from_utf8(&msg).unwrap();
+ msg.contains("--remove-pass")
+}
+
+fn run_creduce<'a>(
+ creduce_cmd: &str,
+ interestingness_test: &'a Path,
+ concat_path: &'a Path,
+ creduce_args: impl Iterator<Item = &'a str>,
+) {
+ announce_progress("creduce");
+ let args = std::iter::once(interestingness_test.to_str().unwrap())
+ .chain(std::iter::once(concat_path.to_str().unwrap()))
+ .chain(creduce_args)
+ .chain(
+ if creduce_supports_remove_pass(creduce_cmd) {
+ REMOVE_PASS_LINE_MARKERS
+ } else {
+ SKIP_INITIAL_PASSES
+ }
+ .iter()
+ .copied(),
+ )
+ .collect::<Vec<_>>();
+ println!("Command: {} {}", creduce_cmd, args.join(" "));
+ std::process::Command::new(creduce_cmd)
+ .args(args)
+ .status()
+ .expect("failed to creduce");
+}
+
+fn run_sample_gen_cmd(
+ gen_cmd: &str,
+ rs_file: &Path,
+ tmp_dir: &Path,
+ extra_clang_args: &[&str],
+) -> Result<(), std::io::Error> {
+ let args = format_gen_cmd(rs_file, tmp_dir.to_str().unwrap(), extra_clang_args);
+ let args = args.collect::<Vec<_>>();
+ let args_str = args.join(" ");
+ announce_progress(&format!("Running sample gen cmd: {} {}", gen_cmd, args_str));
+ std::process::Command::new(gen_cmd).args(args).status()?;
+ Ok(())
+}
+
+fn run_demo_interestingness_test(demo_dir: &Path, test: &Path) -> Result<(), std::io::Error> {
+ announce_progress(&format!(
+ "Running demo interestingness test in {}",
+ demo_dir.to_string_lossy()
+ ));
+ std::process::Command::new(test)
+ .current_dir(demo_dir)
+ .status()?;
+ Ok(())
+}
+
+fn format_gen_cmd<'a>(
+ rs_file: &Path,
+ dir: &str,
+ extra_clang_args: &'a [&str],
+) -> impl Iterator<Item = String> + 'a {
+ let args = [
+ "-o".to_string(),
+ dir.to_string(),
+ "-I".to_string(),
+ dir.to_string(),
+ rs_file.to_str().unwrap().to_string(),
+ "--gen-rs-include".to_string(),
+ "--gen-cpp".to_string(),
+ "--suppress-system-headers".to_string(),
+ "--".to_string(),
+ ]
+ .to_vec();
+ args.into_iter()
+ .chain(extra_clang_args.iter().map(|s| s.to_string()))
+}
+
+fn create_interestingness_test(
+ gen_cmd: &str,
+ test_path: &Path,
+ problem: Option<&str>,
+ rs_file: &Path,
+ extra_clang_args: &[&str],
+ precompile: bool,
+ postcompile: bool,
+) -> Result<(), std::io::Error> {
+ announce_progress("Creating interestingness test");
+ // Ensure we refer to the input header by relative path
+ // because creduce will invoke us in some other directory with
+ // a copy thereof.
+ let mut args = format_gen_cmd(rs_file, "$(pwd)", extra_clang_args);
+ let args = args.join(" ");
+ let precompile_step = make_compile_step(precompile, "concat.h", extra_clang_args);
+ // For the compile afterwards, we have to avoid including any system headers.
+ // We rely on equivalent content being hermetically inside concat.h.
+ let postcompile_step = make_compile_step(postcompile, "gen0.cc", extra_clang_args);
+ let problem_grep = problem
+ .map(|problem| format!("| grep \"{}\" >/dev/null 2>&1", problem))
+ .unwrap_or_default();
+ let content = format!(
+ indoc! {"
+ #!/bin/sh
+ set -e
+ echo Precompile
+ {}
+ echo Move
+ mv concat.h concat-body.h
+ echo Codegen
+ (echo \"#ifndef __CONCAT_H__\"; echo \"#define __CONCAT_H__\"; echo '#include \"concat-body.h\"'; echo \"#endif\") > concat.h
+ ({} {} 2>&1 && cat autocxx-ffi-default-gen.rs && cat autocxxgen*.h && {} 2>&1 ) {}
+ echo Remove
+ rm concat.h
+ echo Swap back
+ mv concat-body.h concat.h
+ echo Done
+ "},
+ precompile_step, gen_cmd, args, postcompile_step, problem_grep
+ );
+ println!("Interestingness test:\n{}", content);
+ {
+ let mut file = File::create(test_path)?;
+ file.write_all(content.as_bytes())?;
+ }
+
+ let mut perms = std::fs::metadata(&test_path)?.permissions();
+ perms.set_mode(0o700);
+ std::fs::set_permissions(&test_path, perms)?;
+ Ok(())
+}
+
+fn make_compile_step(enabled: bool, file: &str, extra_clang_args: &[&str]) -> String {
+ if enabled {
+ format!(
+ "{} {} -c {}",
+ get_clang_path(),
+ make_clang_args(&[PathBuf::from(".")], extra_clang_args).join(" "),
+ file,
+ )
+ } else {
+ "echo 'Skipping compilation'".into()
+ }
+}
+
+fn create_rs_file(rs_path: &Path, directives: &[String]) -> Result<(), std::io::Error> {
+ announce_progress("Creating Rust input file");
+ let mut file = File::create(rs_path)?;
+ file.write_all("use autocxx::include_cpp;\ninclude_cpp! (\n".as_bytes())?;
+ for directive in directives {
+ file.write_all(directive.as_bytes())?;
+ }
+ file.write_all(");\n".as_bytes())?;
+ Ok(())
+}
+
+fn create_concatenated_header(headers: &[&str], listing_path: &Path) -> Result<(), std::io::Error> {
+ announce_progress("Creating preprocessed header");
+ let mut file = File::create(listing_path)?;
+ for header in headers {
+ file.write_all(format!("#include \"{}\"\n", header).as_bytes())?;
+ }
+ Ok(())
+}
+
+fn create_file(path: &Path, content: &str) -> Result<(), std::io::Error> {
+ let mut file = File::create(path)?;
+ write!(file, "{}", content)?;
+ Ok(())
+}
+
+fn get_cxx_suppressions() -> Vec<String> {
+ let defines: IndexSet<_> = Regex::new(r"\bCXXBRIDGE1_\w+\b")
+ .unwrap()
+ .find_iter(cxx_gen::HEADER)
+ .map(|m| m.as_str())
+ .collect(); // for uniqueness
+ defines
+ .into_iter()
+ .map(|def| format!("-D{}", def))
+ .collect()
+}
+
+#[test]
+fn test_get_cxx_suppressions() {
+ let defines = get_cxx_suppressions();
+ assert!(defines.contains(&"-DCXXBRIDGE1_RUST_BITCOPY_T".to_string()));
+ assert!(defines.contains(&"-DCXXBRIDGE1_RUST_STR".to_string()));
+}