Persist fault injection logs and fail fast on expected-state trace writes (#14651)

Summary:
- switch fault injection error recording from an in-memory ring buffer to per-run fixed-record binary logs under `TEST_TMPDIR/fault_injection_logs` (or `/tmp/fault_injection_logs`) so crash paths survive DB reopen cleanup
- keep the raw and decoded fault logs for external artifact collection/cleanup, and make `db_crashtest` print consistent blackbox/whitebox summaries after decoding
- make expected-state tracing fail fast on trace write failures and document offline trace inspection via `trace_analyzer`
- add coverage for binary log persistence/decoding/truncated-tail handling and keep info logs excluded from fault injection

Reviewed By: hx235

Differential Revision: D101973626

fbshipit-source-id: fdcb5b6370cf92a046e09b8d3391e80eecb66c23

Author: xingbowang

db_stress_tool/db_stress_gflags.cc

@@ -706,20 +706,6 @@ DEFINE_string(
     "only tracked when --sync_fault_injection is set. See --seed and "
     "--nooverwritepercent for further requirements.");
 
-DEFINE_bool(expected_state_trace_debug, true,
-            "If true, print debug logs while replaying expected-state trace "
-            "records during crash recovery verification.");
-
-DEFINE_int64(
-    expected_state_trace_debug_key, -1,
-    "If non-negative, restrict expected-state trace debug logs to the "
-    "specified logical key where possible. Raw-key roundtrip mismatches for "
-    "that logical key are still logged.");
-
-DEFINE_int32(expected_state_trace_debug_max_logs, 200,
-             "Maximum number of expected-state trace debug log lines to emit "
-             "per restore attempt.");
-
 DEFINE_bool(verify_checksum, false,
             "Verify checksum for every block read from storage");
 

db_stress_tool/db_stress_test_base.cc

@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
 
+#include <cstdlib>
 #include <ios>
 #include <memory>
 #include <mutex>
@@ -30,6 +31,7 @@
 #include "db_stress_tool/db_stress_wide_merge_operator.h"
 #include "file/file_util.h"
 #include "options/options_parser.h"
+#include "port/port.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/io_dispatcher.h"
@@ -258,6 +260,32 @@ bool NeedsFaultInjection() {
          FLAGS_write_fault_one_in || FLAGS_sync_fault_injection;
 }
 
+std::string GetFaultInjectionLogBaseDir() {
+  if (!FLAGS_env_uri.empty() || !FLAGS_fs_uri.empty()) {
+    return "/tmp";
+  }
+  // db_stress reads environment variables during single-threaded startup,
+  // before worker threads are created.
+  // NOLINTNEXTLINE(concurrency-mt-unsafe)
+  const char* test_tmpdir = std::getenv("TEST_TMPDIR");
+  return test_tmpdir != nullptr && test_tmpdir[0] != '\0' ? test_tmpdir
+                                                          : "/tmp";
+}
+
+std::string GetFaultInjectionLogPath(const std::string& db_label) {
+  const std::string log_dir =
+      GetFaultInjectionLogBaseDir() + "/fault_injection_logs";
+  Status s = Env::Default()->CreateDirIfMissing(log_dir);
+  if (!s.ok()) {
+    fprintf(stderr, "Failed to create directory %s: %s\n", log_dir.c_str(),
+            s.ToString().c_str());
+    exit(1);
+  }
+  return log_dir + "/fault_injection_" + std::to_string(port::GetProcessID()) +
+         "_" + std::to_string(Env::Default()->NowMicros()) + "_" + db_label +
+         ".bin";
+}
+
 }  // namespace
 
 const std::string& StressTest::GetDbLabel() const { return db_label_; }
@@ -285,7 +313,8 @@ StressTest::StressTest(int db_index, const std::string& db_path,
           std::make_shared<DbStressFSWrapper>(raw_env->GetFileSystem())),
       db_fault_injection_fs_(
           NeedsFaultInjection()
-              ? std::make_shared<FaultInjectionTestFS>(db_stress_fs_)
+              ? std::make_shared<FaultInjectionTestFS>(
+                    db_stress_fs_, GetFaultInjectionLogPath(db_label_))
               : nullptr),
       db_env_(std::make_unique<CompositeEnvWrapper>(
           raw_env, db_fault_injection_fs_
@@ -314,23 +343,6 @@ StressTest::StressTest(int db_index, const std::string& db_path,
     // StressTest::Open() for open fault injection and in RunStressTestImpl()
     // for proper fault injection setup.
     db_fault_injection_fs_->SetFilesystemDirectWritable(true);
-
-    // Set the fault injection log file path so that PrintAll() writes to a
-    // file instead of stderr (signal-safe). PrintAll() opens this path with
-    // plain POSIX open(), not through raw_env, so the log path must stay on
-    // the local filesystem. Store it in the local test directory (TEST_TMPDIR
-    // via Env::Default()) outside the DB directory so it survives DB reopen
-    // and gets included in the sandcastle db.tar.gz artifact for post-failure
-    // analysis.
-    std::string log_dir;
-    if (!Env::Default()->GetTestDirectory(&log_dir).ok() || log_dir.empty()) {
-      log_dir = "/tmp";
-    }
-    std::string log_path = log_dir + "/fault_injection_" +
-                           std::to_string(getpid()) + "_" +
-                           std::to_string(Env::Default()->NowMicros()) + "_" +
-                           GetDbLabel() + ".log";
-    db_fault_injection_fs_->SetInjectedErrorLogPath(log_path);
   }
 
   if (FLAGS_destroy_db_initially) {

db_stress_tool/db_stress_tool.cc

@@ -38,7 +38,7 @@ static std::shared_ptr<ROCKSDB_NAMESPACE::Env> legacy_env_wrapper_guard;
 // Raw pointers for signal-safe crash callback. Signal handlers can only
 // access file-static/global variables; can't capture StressTest instances.
 static std::vector<ROCKSDB_NAMESPACE::FaultInjectionTestFS*>
-    fault_fs_for_crash_report;
+    fault_fs_for_crash_flush;
 
 int ValidateNumDbsFlags() {
   if (FLAGS_num_dbs < 1) {
@@ -88,20 +88,20 @@ int DestroyAllDbs() {
 
 void RegisterCrashCallbacks(
     const std::vector<std::unique_ptr<StressTest>>& stress_tests, int num_dbs) {
-  fault_fs_for_crash_report.resize(num_dbs, nullptr);
+  fault_fs_for_crash_flush.resize(num_dbs, nullptr);
   bool any_fault_fs = false;
   for (int i = 0; i < num_dbs; i++) {
-    fault_fs_for_crash_report[i] =
+    fault_fs_for_crash_flush[i] =
         stress_tests[i]->GetDbFaultInjectionFs().get();
-    if (fault_fs_for_crash_report[i]) {
+    if (fault_fs_for_crash_flush[i]) {
       any_fault_fs = true;
     }
   }
   if (any_fault_fs) {
     port::RegisterCrashCallback([]() {
-      for (auto* fs : fault_fs_for_crash_report) {
+      for (auto* fs : fault_fs_for_crash_flush) {
         if (fs) {
-          fs->PrintRecentInjectedErrors();
+          fs->FlushRecentInjectedErrors();
         }
       }
     });
@@ -633,7 +633,7 @@ int db_stress_tool(int argc, char** argv) {
     }
     stress_tests[i]->CleanUp();
   }
-  for (auto& fs : fault_fs_for_crash_report) {
+  for (auto& fs : fault_fs_for_crash_flush) {
     fs = nullptr;
   }
   return all_passed ? 0 : 1;