nvl32: Add nvl32 target
Also adds an i2c device class to allow us to send raw i2c
Tested:
ran on the nvl32-obmc model and i2c devices were enumerated correctly.
as well as mctp devices
Change-Id: I073156de2bbe06b06017379de35e076166df3875
Signed-off-by: Marc Olberding <molberding@nvidia.com>
diff --git a/i2c.cpp b/i2c.cpp
index e6d60e1..cc8be13 100644
--- a/i2c.cpp
+++ b/i2c.cpp
@@ -3,9 +3,22 @@
#include "i2c.hpp"
+#include <fcntl.h>
+#include <linux/i2c-dev.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#include <filesystem>
#include <format>
#include <fstream>
#include <iostream>
+
+extern "C"
+{
+#include <i2c/smbus.h>
+#include <linux/i2c-dev.h>
+}
+
namespace i2c
{
@@ -68,4 +81,44 @@
std::cerr << std::format("{} device created at bus {}", device_type, bus);
}
+
+RawDevice::RawDevice(size_t bus, uint8_t address)
+{
+ std::string bus_path = std::format("/dev/i2c-{}", bus);
+ std::filesystem::path dev_path = bus_path;
+ fd = open(dev_path.c_str(), O_RDWR);
+ if (fd < 0)
+ {
+ std::cerr << std::format("failed to open {}\n", dev_path.native());
+ throw std::runtime_error(
+ std::format("Failed to open {}", dev_path.native()));
+ }
+
+ if (ioctl(fd, I2C_SLAVE, address) < 0)
+ {
+ // dtor won't be called since we never finished constructing it, clean
+ // up our fd
+ close(fd);
+ throw std::runtime_error(
+ std::format("Failed to specify address {}", address));
+ }
+}
+
+RawDevice::~RawDevice()
+{
+ close(fd);
+}
+
+int RawDevice::read_byte(uint8_t reg, uint8_t& val)
+{
+ int result = i2c_smbus_read_byte_data(fd, reg);
+ if (result < 0)
+ {
+ return -result;
+ }
+
+ val = result;
+ return 0;
+}
+
} // namespace i2c
diff --git a/i2c.hpp b/i2c.hpp
index 6685162..4c62799 100644
--- a/i2c.hpp
+++ b/i2c.hpp
@@ -3,6 +3,7 @@
#pragma once
+#include <cstdint>
#include <string>
namespace i2c
@@ -11,4 +12,19 @@
void rebind_controller(const std::string_view number);
void new_device(unsigned int bus, unsigned int address,
std::string_view device_type);
+
+// a simple RAII wrapper for raw i2c comms
+struct RawDevice
+{
+ RawDevice(size_t bus, uint8_t address);
+ ~RawDevice();
+ RawDevice(const RawDevice&) = delete;
+ RawDevice& operator=(const RawDevice&) = delete;
+ RawDevice& operator=(RawDevice&&) = default;
+ RawDevice(RawDevice&&) = default;
+
+ int read_byte(uint8_t reg, uint8_t& val);
+
+ int fd;
+};
} // namespace i2c
diff --git a/meson.build b/meson.build
index 722019f..63b8773 100644
--- a/meson.build
+++ b/meson.build
@@ -11,14 +11,15 @@
libsystemd_dep = dependency('libsystemd')
cli11_dep = dependency('CLI11', required: true, include_type: 'system')
+i2c_dep = meson.get_compiler('cpp').find_library('i2c')
-platform_srcs = files('nvidia/gb200.cpp')
+platform_srcs = files('nvidia/gb200.cpp', 'nvidia/nvl32.cpp')
util_srcs = files('gpio.cpp', 'i2c.cpp', 'utilities.cpp')
exe = executable(
'platform',
['platform.cpp'] + platform_srcs + util_srcs,
- dependencies: [gpiodcxx_dep, libsystemd_dep, cli11_dep],
+ dependencies: [gpiodcxx_dep, libsystemd_dep, cli11_dep, i2c_dep],
include_directories: ['.', 'nvidia'],
install: true,
install_dir: get_option('libexecdir'),
diff --git a/nvidia/nvidia.hpp b/nvidia/nvidia.hpp
index 83d0686..81c01b1 100644
--- a/nvidia/nvidia.hpp
+++ b/nvidia/nvidia.hpp
@@ -9,5 +9,6 @@
// list your platform initialization callbacks here
int init_gb200_base();
int init_gb200_with_p2020();
+int init_nvl32();
} // namespace nvidia
diff --git a/nvidia/nvl32.cpp b/nvidia/nvl32.cpp
new file mode 100644
index 0000000..0dde0d8
--- /dev/null
+++ b/nvidia/nvl32.cpp
@@ -0,0 +1,247 @@
+#include "gpio.hpp"
+#include "i2c.hpp"
+#include "utilities.hpp"
+
+#include <systemd/sd-daemon.h>
+
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <thread>
+
+namespace nvidia
+{
+
+using steady_clock = std::chrono::steady_clock;
+using namespace std::chrono_literals;
+
+void logged_system(std::string_view cmd)
+{
+ std::cerr << std::format("calling {} \n", cmd);
+ int rc = std::system(cmd.data());
+ (void)rc;
+}
+
+void setup_devmem()
+{
+ logged_system("mknod /dev/mem c 1 1");
+}
+
+void handle_passthrough_registers(bool enable)
+{
+ static constexpr uint32_t reg = 0x1e6e24bc;
+ std::string command;
+ if (enable)
+ {
+ command = std::format("devmem 0x{:x} 32 0x3f000000", reg);
+ }
+ else
+ {
+ command = std::format("devmem 0x{:x} 32 0", reg);
+ }
+ logged_system(command);
+}
+
+void wait_for_i2c_ready()
+{
+ // hpm cpld is at bus 4, address 0x17
+ i2c::RawDevice cpld{4, 0x17};
+ auto now = steady_clock::now();
+ auto end = now + 20min;
+ while (steady_clock::now() < end)
+ {
+ static constexpr uint8_t i2c_ready = 0xf2;
+ uint8_t result;
+ int rc = cpld.read_byte(i2c_ready, result);
+ if (rc)
+ {
+ std::string err =
+ std::format("Unable to communicate with cpld. rc: {}\n", rc);
+ std::cerr << err;
+ throw std::runtime_error(err);
+ }
+
+ if (result == 1)
+ {
+ return;
+ }
+
+ std::this_thread::sleep_for(std::chrono::seconds{10});
+ }
+
+ throw std::runtime_error("Waiting for host timed out!\n");
+}
+
+void probe_dev(size_t bus, uint8_t address, std::string_view dev_type)
+{
+ std::filesystem::path path =
+ std::format("/sys/bus/i2c/devices/i2c-{}/new_device", bus);
+
+ std::ofstream f{path};
+ if (!f.good())
+ {
+ std::cerr << std::format("Unable to open {}\n", path.c_str());
+ std::exit(EXIT_FAILURE);
+ }
+
+ f << std::format("{} 0x{:02x}", dev_type, address);
+ f.close();
+
+ std::string created_path =
+ std::format("/sys/bus/i2c/devices/{}-{:04x}", bus, address);
+ wait_for_path_to_exist(created_path, 10ms);
+}
+
+void create_i2c_mux(size_t bus, uint8_t address, std::string_view dev_type)
+{
+ probe_dev(bus, address, dev_type);
+
+ std::filesystem::path idle =
+ std::format("/sys/bus/i2c/devices/{}-{:04x}/idle_state", bus, address);
+ std::ofstream idle_f{idle};
+ if (!idle_f.good())
+ {
+ std::string err = std::format("Unable to open {}\n", idle.c_str());
+ std::cerr << err;
+ throw std::runtime_error(err);
+ }
+
+ // -2 is idle-mux-disconnect
+ idle_f << -2;
+ idle_f.close();
+}
+
+size_t get_bus_from_channel(size_t parent_bus, uint8_t address, size_t channel)
+{
+ std::filesystem::path path =
+ std::format("/sys/bus/i2c/devices/{}-{:04x}/channel-{}/i2c-dev/",
+ parent_bus, address, channel);
+ int bus = -1;
+ std::error_code ec{};
+ for (const auto& f : std::filesystem::directory_iterator(path, ec))
+ {
+ // we expect to see i2c-<bus>, trim and parse everything after the dash
+ const std::string& p = f.path().filename().string();
+ std::cerr << "Reading from " << p << "\n";
+ auto [_, err] = std::from_chars(p.data() + 4, p.data() + p.size(), bus);
+ if (err != std::errc{})
+ {
+ std::string err_s = std::format("Failed to parse {}\n", p);
+ std::cerr << err_s;
+ throw std::runtime_error(err_s);
+ }
+ }
+ if (bus == -1 || ec)
+ {
+ std::string err_s =
+ std::format("Failed to find a channel at {}\n", path.string());
+ std::cerr << err_s;
+ throw std::runtime_error(err_s);
+ }
+ return bus;
+}
+
+void bringup_cx8_mcu(size_t bus)
+{
+ probe_dev(bus, 0x26, "pca9555");
+ std::string gpio_p =
+ std::format("/sys/bus/i2c/devices/{}-{:04x}/", bus, 0x26);
+ int chip_num = gpio::find_chip_idx_from_dir(gpio_p);
+ if (chip_num < 0)
+ {
+ std::cerr << std::format("Failed to find cx8 gpio at {}\n", gpio_p);
+ std::exit(EXIT_FAILURE);
+ }
+
+ // 14 is the reset pin on the MCU
+ // reset pin is active low
+ gpio::set_raw(chip_num, 14, 1);
+}
+
+void gringup_gpu_sma(size_t bus, size_t channel)
+{
+ size_t gpu_bus = get_bus_from_channel(bus, 0x72, channel);
+ probe_dev(gpu_bus, 0x20, "pca6408");
+ std::string gpio_p =
+ std::format("/sys/bus/i2c/devices/{}-{:04x}/", gpu_bus, 0x20);
+ int chip_num = gpio::find_chip_idx_from_dir(gpio_p);
+ if (chip_num < 0)
+ {
+ std::cerr << std::format("Failed to find gpu gpio {}\n", gpio_p);
+ std::exit(EXIT_FAILURE);
+ }
+
+ // pin 4 is the reset pin, active low
+ // pin 5 engages the telemetry path from the SMA
+ gpio::set_raw(chip_num, 5, 1);
+ gpio::set_raw(chip_num, 4, 1);
+}
+
+void bringup_gpus_on_mcio(size_t bus)
+{
+ create_i2c_mux(bus, 0x72, "pca9546");
+
+ gringup_gpu_sma(bus, 2);
+ gringup_gpu_sma(bus, 3);
+}
+
+void bringup_cx8_mcio(size_t mux_addr, size_t channel, bool has_cx8)
+{
+ size_t bus = get_bus_from_channel(5, mux_addr, channel);
+ if (has_cx8)
+ {
+ bringup_cx8_mcu(bus);
+ }
+ bringup_gpus_on_mcio(bus);
+}
+
+void enumerate_mctp(int dev_num)
+{
+ // TODO: Make this a proper dbus client
+ std::string preamble = "busctl call au.com.codeconstruct.MCTP1";
+ std::string postamble =
+ "au.com.codeconstruct.MCTP1.BusOwner1 AssignEndpoint ay 0";
+
+ std::string cmd =
+ std::format("{} /au/com/codeconstruct/mctp1/interfaces/mctpusb{} {}",
+ preamble, dev_num, postamble);
+ logged_system(cmd);
+}
+
+void wait_for_usb_to_probe()
+{
+ std::this_thread::sleep_for(std::chrono::seconds{20});
+}
+
+int init_nvl32()
+{
+ setup_devmem();
+ handle_passthrough_registers(false);
+ sd_notify(0, "READY=1");
+
+ wait_for_i2c_ready();
+
+ create_i2c_mux(5, 0x70, "pca9548");
+ create_i2c_mux(5, 0x71, "pca9548");
+ create_i2c_mux(5, 0x73, "pca9548");
+ create_i2c_mux(5, 0x75, "pca9548");
+
+ bringup_cx8_mcio(0x70, 1, true);
+ bringup_cx8_mcio(0x70, 5, false);
+ bringup_cx8_mcio(0x73, 3, true);
+ bringup_cx8_mcio(0x73, 7, false);
+
+ wait_for_usb_to_probe();
+ for (int ctr = 0; ctr < 10; ++ctr)
+ {
+ enumerate_mctp(ctr);
+ }
+ std::cerr << "platform init complete\n";
+ pause();
+ std::cerr << "Releasing platform\n";
+
+ return EXIT_SUCCESS;
+}
+
+} // namespace nvidia
diff --git a/platform.cpp b/platform.cpp
index 15f4948..bce78bc 100644
--- a/platform.cpp
+++ b/platform.cpp
@@ -18,9 +18,10 @@
#include <string_view>
#include <utility>
-constexpr std::array<std::pair<std::string_view, int (*)()>, 2> init_functions{
+constexpr std::array<std::pair<std::string_view, int (*)()>, 3> init_functions{
{{"nvidia-gb200", nvidia::init_gb200_base},
- {"nvidia-gb200-with-p2020", nvidia::init_gb200_with_p2020}}};
+ {"nvidia-gb200-with-p2020", nvidia::init_gb200_with_p2020},
+ {"nvidia-nvl32", nvidia::init_nvl32}}};
int main(int argc, char** argv)
{