[Celestica] Leh800bcls: Resolve multi-switch empty fb303 counters in IS_OSS Split Agent mode

Author: gang-tao

fboss/agent/hw/test/HwTestThriftHandler.h

@@ -10,6 +10,8 @@
 
 #pragma once
 
+#include <fb303/ServiceData.h>
+#include <fb303/ThreadCachedServiceData.h>
 #include "fboss/agent/if/gen-cpp2/AgentHwTestCtrl.h"
 
 #include "fboss/agent/FbossError.h"
@@ -258,6 +260,18 @@ class HwTestThriftHandler : public AgentHwTestCtrlSvIf {
       std::vector<std::string>& out,
       std::unique_ptr<std::string> substring) override;
 
+  void getFb303RegexCounters(
+      std::map<std::string, int64_t>& counters,
+      std::unique_ptr<std::string> regex) override {
+    facebook::fb303::ThreadCachedServiceData::get()->publishStats();
+    counters = facebook::fb303::fbData->getRegexCounters(*regex);
+  }
+
+  int64_t getFb303Counter(std::unique_ptr<std::string> key) override {
+    facebook::fb303::ThreadCachedServiceData::get()->publishStats();
+    return facebook::fb303::fbData->getCounterIfExists(*key).value_or(0);
+  }
+
  private:
   HwSwitch* hwSwitch_;
   // Captures log messages in this process for tests to read over RPC.

fboss/agent/if/agent_hw_test_ctrl.thrift

@@ -240,4 +240,8 @@ service AgentHwTestCtrl {
   // installLogCapture() must be called before the log is emitted.
   void installLogCapture();
   list<string> getMatchingLogMessages(1: string substring);
+
+  // fb303 cross-process utils for multi-switch testing
+  map<string, i64> getFb303RegexCounters(1: string regex);
+  i64 getFb303Counter(1: string key);
 }

fboss/agent/test/AgentEnsemble.cpp

@@ -881,8 +881,20 @@ std::map<std::string, int64_t> AgentEnsemble::getFb303CountersByRegex(
       client->getChannelShared()};
   monitoringClient.sync_getRegexCounters(counters, regex);
 #else
-  // TODO: This needs to be updated to support multi-switch.
-  counters = facebook::fb303::fbData->getRegexCounters(regex);
+  if (!getSw()->isRunModeMultiSwitch()) {
+    counters = facebook::fb303::fbData->getRegexCounters(regex);
+  } else {
+    try {
+      auto switchID = scopeResolver().scope(portId).switchId();
+      auto hwTestClient = getHwAgentTestClient(switchID);
+      hwTestClient->sync_getFb303RegexCounters(counters, regex);
+    } catch (const std::exception& ex) {
+      XLOG(WARNING)
+          << "Failed to fetch remote fb303 counters via HwTestCtrl for port "
+          << portId << ": " << ex.what();
+      counters = facebook::fb303::fbData->getRegexCounters(regex);
+    }
+  }
 #endif
   return counters;
 }
@@ -908,8 +920,19 @@ int64_t AgentEnsemble::getFb303Counter(
       client->getChannelShared()};
   counter = monitoringClient.sync_getCounter(key);
 #else
-  // TODO: This needs to be updated to support multi-switch.
-  counter = facebook::fb303::fbData->getCounter(key);
+  if (!getSw()->isRunModeMultiSwitch()) {
+    counter = facebook::fb303::fbData->getCounterIfExists(key).value_or(0);
+  } else {
+    try {
+      auto hwTestClient = getHwAgentTestClient(switchID);
+      counter = hwTestClient->sync_getFb303Counter(key);
+    } catch (const std::exception& ex) {
+      XLOG(WARNING)
+          << "Failed to fetch remote fb303 counter via HwTestCtrl for switch "
+          << switchID << ": " << ex.what();
+      counter = facebook::fb303::fbData->getCounterIfExists(key).value_or(0);
+    }
+  }
 #endif
   return counter;
 }
@@ -959,8 +982,19 @@ std::map<std::string, int64_t> AgentEnsemble::getFb303RegexCounters(
       client->getChannelShared()};
   monitoringClient.sync_getRegexCounters(counters, regex);
 #else
-  // TODO: This needs to be updated to support multi-switch.
-  counters = facebook::fb303::fbData->getRegexCounters(regex);
+  if (!getSw()->isRunModeMultiSwitch()) {
+    counters = facebook::fb303::fbData->getRegexCounters(regex);
+  } else {
+    try {
+      auto hwTestClient = getHwAgentTestClient(switchID);
+      hwTestClient->sync_getFb303RegexCounters(counters, regex);
+    } catch (const std::exception& ex) {
+      XLOG(WARNING)
+          << "Failed to fetch remote fb303 counters via HwTestCtrl for switch "
+          << switchID << ": " << ex.what();
+      counters = facebook::fb303::fbData->getRegexCounters(regex);
+    }
+  }
 #endif
   return counters;
 }