diff --git a/README.md b/README.md index 909b621..99ab8ae 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,58 @@ You also can mount your own `config.yaml` to override the default settings: docker run --network=host -it -v $(pwd)/my_config.yaml:/app/config.yaml cloud-point-rpc ``` +## Validation with SCARED Dataset + +The `scared_dataset_server` executable lets you validate the stereo point-cloud +pipeline against real endoscopic images from the +[SCARED dataset](https://huggingface.co/datasets/maxhallan7/scared). + +### Obtaining the data + +1. Download `test_dataset_8.zip` from + . +2. Extract so that `keyframe_0/` through `keyframe_4/` exist under + `test_dataset_8/`. + +Each keyframe directory contains: +- `Left_Image.png`, `Right_Image.png` — 1280×1024 unrectified RGBA images. +- `endoscope_calibration.yaml` — OpenCV FileStorage with `M1`, `D1`, `M2`, + `D2`, `R`, `T` nodes. + +**Note:** `T` is stored in **millimetres** in the YAML file (baseline ≈ −4.35 mm). +`scared_dataset_server` divides `T` by 1000 before placing it on the wire +(the wire protocol uses metres). + +### Running the server + +```bash +./build/src/cloud_point/scared_dataset_server \ + /path/to/test_dataset_8/keyframe_0 8080 +``` + +### Connecting with the CLI + +In a second terminal run the interactive CLI against the same host and port: + +```bash +# Adjust ip/port in config.yaml if needed, then: +./build/src/cloud_point_rpc_cli config.yaml +# Option 4 — compute point cloud and print valid point count +# Option 5 — compute point cloud and save to output.ply (inspect in MeshLab) +``` + +The SCARED test set contains no ground-truth depth, so validation is +qualitative (inspect the PLY in MeshLab or similar). + +**Disparity range caveat:** the CLI constructs `CloudPointClient` with the +default of 128 disparity levels, while this rig (fx ≈ 1024 px, baseline +≈ 4.35 mm) produces disparities above 128 px for tissue nearer than +~35 mm — those pixels silently drop out of the cloud. The E2E test +(`tests/test_scared_dataset.cpp`) passes `num_disparities = 160` for full +coverage; programmatic consumers should do the same via the +`CloudPointClient` constructor. For the CLI's qualitative check the default +is fine (observed median depth ≈ 115 mm is well within range). + ## Communication model ![Communicatoin model plantuml diagram](docs/cm.png) diff --git a/include/cloud_point/scared_dataset_loader.hpp b/include/cloud_point/scared_dataset_loader.hpp new file mode 100644 index 0000000..bbf1d15 --- /dev/null +++ b/include/cloud_point/scared_dataset_loader.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "cloud_point_rpc/rpc_dto.hpp" +#include + +namespace score { + +/// @brief Loads stereo calibration and image pair from a SCARED dataset +/// keyframe directory. +/// +/// Expected directory layout: +/// /endoscope_calibration.yaml — OpenCV FileStorage +/// /Left_Image.png — 1280x1024 RGBA PNG +/// /Right_Image.png — 1280x1024 RGBA PNG +/// +/// The YAML node T is in millimetres; this loader converts to metres before +/// populating StereoCalibrationRPC.translation. +class ScaredDatasetLoader { + public: + /// @brief Load calibration and images from @p keyframe_dir. + /// @throws std::runtime_error if any file cannot be opened or parsed. + explicit ScaredDatasetLoader(const std::string &keyframe_dir); + + /// @brief Return the stereo calibration DTO (translation in metres). + [[nodiscard]] const StereoCalibrationRPC &calibration() const noexcept; + + /// @brief Return an image pair DTO with the given frame index. + /// The images are the same for every call (single keyframe). + [[nodiscard]] ImagePairRPC image_pair(uint64_t frame) const; + + private: + StereoCalibrationRPC calib_; + ImageRPC left_image_; + ImageRPC right_image_; +}; + +} // namespace score diff --git a/src/cloud_point/meson.build b/src/cloud_point/meson.build index 2fbbeba..6101343 100644 --- a/src/cloud_point/meson.build +++ b/src/cloud_point/meson.build @@ -19,6 +19,7 @@ cloud_point_sources = files( 'stereo_rectifier.cpp', 'point_cloud_builder.cpp', 'cloud_point_client.cpp', + 'scared_dataset_loader.cpp', ) cpc_deps = [ cloud_point_rpc_dep, opencv_dep ] @@ -40,3 +41,10 @@ cloud_point_compute_dep = declare_dependency( link_with: cloud_point_compute_lib, dependencies: cpc_deps ) + +executable( + 'scared_dataset_server', + 'scared_dataset_server.cpp', + dependencies: [cloud_point_compute_dep], + install: true, +) diff --git a/src/cloud_point/scared_dataset_loader.cpp b/src/cloud_point/scared_dataset_loader.cpp new file mode 100644 index 0000000..f5358a6 --- /dev/null +++ b/src/cloud_point/scared_dataset_loader.cpp @@ -0,0 +1,117 @@ +#include "cloud_point/scared_dataset_loader.hpp" + +#include +#include +#include + +#include +#include +#include + +namespace score { + +namespace { + +/// Flatten a cv::Mat (row-major) to a std::vector. +std::vector mat_to_vec(const cv::Mat &m) { + cv::Mat d64; + m.convertTo(d64, CV_64F); + std::vector v(static_cast(d64.total())); + std::copy(d64.begin(), d64.end(), v.begin()); + return v; +} + +/// Load a PNG from @p path and encode as BGR ImageRPC. +/// Reads with IMREAD_COLOR (→ BGR 8-bit); throws on failure. +ImageRPC load_bgr_image(const std::string &path) { + const cv::Mat img = cv::imread(path, cv::IMREAD_COLOR); + if (img.empty()) { + throw std::runtime_error( + "ScaredDatasetLoader: cannot read image: " + path); + } + ImageRPC rpc; + rpc.width = img.cols; + rpc.height = img.rows; + rpc.type = ImageRPC::Type::BGR; + const size_t sz = static_cast(img.cols) * img.rows * 3; + rpc.data.resize(sz); + std::memcpy(rpc.data.data(), img.data, sz); + return rpc; +} + +} // namespace + +ScaredDatasetLoader::ScaredDatasetLoader(const std::string &keyframe_dir) { + const std::string yaml_path = keyframe_dir + "/endoscope_calibration.yaml"; + const std::string left_path = keyframe_dir + "/Left_Image.png"; + const std::string right_path = keyframe_dir + "/Right_Image.png"; + + LOG(INFO) << "ScaredDatasetLoader: loading calibration from " << yaml_path; + + cv::FileStorage fs(yaml_path, cv::FileStorage::READ); + if (!fs.isOpened()) { + throw std::runtime_error( + "ScaredDatasetLoader: cannot open calibration YAML: " + yaml_path); + } + + cv::Mat M1, D1, M2, D2, R, T; + fs["M1"] >> M1; + fs["D1"] >> D1; + fs["M2"] >> M2; + fs["D2"] >> D2; + fs["R"] >> R; + fs["T"] >> T; + fs.release(); + + if (M1.empty() || D1.empty() || M2.empty() || D2.empty() || R.empty() || + T.empty()) { + throw std::runtime_error( + "ScaredDatasetLoader: missing node in calibration YAML: " + + yaml_path); + } + + // Load images first so we can populate width/height from actual dimensions. + LOG(INFO) << "ScaredDatasetLoader: loading images"; + left_image_ = load_bgr_image(left_path); + right_image_ = load_bgr_image(right_path); + + calib_.width = left_image_.width; + calib_.height = left_image_.height; + + calib_.left.camera_matrix = mat_to_vec(M1); + calib_.left.dist_coeffs = mat_to_vec(D1); + calib_.right.camera_matrix = mat_to_vec(M2); + calib_.right.dist_coeffs = mat_to_vec(D2); + calib_.rotation = mat_to_vec(R); + + // T is stored as 1x3 in millimetres; convert to metres. + const auto t_mm = mat_to_vec(T); + calib_.translation.resize(3); + calib_.translation[0] = t_mm[0] / 1000.0; + calib_.translation[1] = t_mm[1] / 1000.0; + calib_.translation[2] = t_mm[2] / 1000.0; + + LOG(INFO) << "ScaredDatasetLoader: T(mm)=[" + << t_mm[0] << "," << t_mm[1] << "," << t_mm[2] + << "] -> T(m)=[" + << calib_.translation[0] << "," + << calib_.translation[1] << "," + << calib_.translation[2] << "]"; + LOG(INFO) << "ScaredDatasetLoader: image size " + << calib_.width << "x" << calib_.height; +} + +const StereoCalibrationRPC & +ScaredDatasetLoader::calibration() const noexcept { + return calib_; +} + +ImagePairRPC ScaredDatasetLoader::image_pair(uint64_t frame) const { + ImagePairRPC pair; + pair.frame = frame; + pair.left = left_image_; + pair.right = right_image_; + return pair; +} + +} // namespace score diff --git a/src/cloud_point/scared_dataset_server.cpp b/src/cloud_point/scared_dataset_server.cpp new file mode 100644 index 0000000..109fecf --- /dev/null +++ b/src/cloud_point/scared_dataset_server.cpp @@ -0,0 +1,73 @@ +/// @file scared_dataset_server.cpp +/// @brief RPC server backed by a SCARED dataset keyframe directory. +/// +/// Usage: scared_dataset_server [port] +/// +/// Serves get-stereo-calibration and get-image-pair matching the wire +/// protocol consumed by CloudPointClient. The same images are returned on +/// every get-image-pair call (single-keyframe source); the frame counter +/// increments so the client can detect stale frames if desired. +#include "cloud_point/scared_dataset_loader.hpp" +#include "cloud_point_rpc/rpc_dto.hpp" +#include "cloud_point_rpc/rpc_server.hpp" +#include "cloud_point_rpc/tcp_server.hpp" +#include +#include +#include + +using json = nlohmann::json; + +int main(int argc, char *argv[]) { + google::InitGoogleLogging(argv[0]); + google::InstallFailureSignalHandler(); + FLAGS_alsologtostderr = 1; + + if (argc < 2) { + LOG(ERROR) << "Usage: " << argv[0] << " [port]"; + return 1; + } + + const std::string keyframe_dir = argv[1]; + const int port = (argc >= 3) ? std::stoi(argv[2]) : 8080; + + LOG(INFO) << "SCARED dataset server starting"; + LOG(INFO) << " keyframe_dir = " << keyframe_dir; + LOG(INFO) << " port = " << port; + + try { + score::ScaredDatasetLoader loader(keyframe_dir); + + uint64_t frame_counter = 0; + score::RpcServer rpc_server; + + rpc_server.register_method( + "get-stereo-calibration", [&](const json &) -> json { + json j; + score::to_json(j, loader.calibration()); + return j; + }); + + rpc_server.register_method( + "get-image-pair", [&](const json &) -> json { + json j; + score::to_json(j, loader.image_pair(frame_counter++)); + return j; + }); + + score::TcpServer server( + "0.0.0.0", port, + [&](const std::string &request) { + return rpc_server.process(request); + }); + + server.start(); + LOG(INFO) << "SCARED dataset server ready on port " << port; + server.join(); + + } catch (const std::exception &e) { + LOG(ERROR) << "Fatal error: " << e.what(); + return 1; + } + + return 0; +} diff --git a/tests/meson.build b/tests/meson.build index 12e224b..2c63c04 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -18,7 +18,8 @@ if opencv_dep.found() 'test_stereo_matcher.cpp', 'test_stereo_rectifier.cpp', 'test_point_cloud_builder.cpp', - 'test_cloud_point_client.cpp' + 'test_cloud_point_client.cpp', + 'test_scared_dataset.cpp' ) test_deps += [cloud_point_compute_dep] else diff --git a/tests/test_scared_dataset.cpp b/tests/test_scared_dataset.cpp new file mode 100644 index 0000000..bab9c93 --- /dev/null +++ b/tests/test_scared_dataset.cpp @@ -0,0 +1,146 @@ +/// @file test_scared_dataset.cpp +/// @brief E2E test: in-process server backed by SCARED dataset + CloudPointClient. +/// +/// Skipped unless env var SCARED_KEYFRAME_DIR is set (CI has no dataset). +/// Run locally: +/// SCARED_KEYFRAME_DIR=/path/to/test_dataset_8/keyframe_0 \ +/// ./build/tests/unit_tests --gtest_filter=ScaredDataset* +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cloud_point/cloud_point_client.hpp" +#include "cloud_point/scared_dataset_loader.hpp" +#include "cloud_point_rpc/rpc_dto.hpp" +#include "cloud_point_rpc/rpc_server.hpp" +#include "cloud_point_rpc/tcp_server.hpp" + +using namespace score; +using json = nlohmann::json; + +// --------------------------------------------------------------------------- +// Fixture: in-process TcpServer + RpcServer backed by the SCARED loader +// --------------------------------------------------------------------------- + +class ScaredDatasetTest : public ::testing::Test { + protected: + void SetUp() override { + FLAGS_logtostderr = true; + if (!google::IsGoogleLoggingInitialized()) + google::InitGoogleLogging("TestScaredDataset"); + + const char *env = std::getenv("SCARED_KEYFRAME_DIR"); + if (!env || std::string(env).empty()) { + GTEST_SKIP() << "SCARED_KEYFRAME_DIR not set; " + "skipping SCARED E2E test"; + } + keyframe_dir_ = env; + } + + void TearDown() override { + if (server_) { + server_->stop(); + } + } + + void start_server(int port, std::unique_ptr rpc) { + rpc_server_ = std::move(rpc); + server_ = std::make_unique( + "127.0.0.1", port, + [this](const std::string &req) { + return rpc_server_->process(req); + }); + server_->start(); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + std::string keyframe_dir_; + std::unique_ptr rpc_server_; + std::unique_ptr server_; +}; + +// --------------------------------------------------------------------------- +// Test: cloud non-empty and median z within plausible endoscopy range +// --------------------------------------------------------------------------- + +TEST_F(ScaredDatasetTest, ComputeCloudFromRealData) { + constexpr int kPort = 9301; + // SCARED rig: fx~1024, B~4.35 mm -> max disparity ~160 needed + constexpr int kNumDisparities = 160; + // Expected depth range for endoscopy: 20 mm - 200 mm + constexpr float kMinExpectedZ = 0.02f; + constexpr float kMaxExpectedZ = 0.20f; + // Minimum valid points for a non-trivial cloud + constexpr size_t kMinValidPts = 50'000; + + ScaredDatasetLoader loader(keyframe_dir_); + + uint64_t frame_counter = 0; + auto rpc = std::make_unique(); + + rpc->register_method( + "get-stereo-calibration", [&](const json &) -> json { + json j; + to_json(j, loader.calibration()); + return j; + }); + rpc->register_method( + "get-image-pair", [&](const json &) -> json { + json j; + to_json(j, loader.image_pair(frame_counter++)); + return j; + }); + + start_server(kPort, std::move(rpc)); + + CloudPointClient client("127.0.0.1", kPort, StereoAlgorithmType::CPU, + PointCloudBuilder::Options{}, kNumDisparities); + ASSERT_NO_THROW(client.connect()); + ASSERT_TRUE(client.connected()); + + auto result = client.compute_cloud(); + ASSERT_TRUE(result.has_value()) + << "compute_cloud returned Error: " << result.error().message; + + const auto &cloud = *result; + const auto valid_points = cloud.valid_points(); + + EXPECT_GE(valid_points.size(), kMinValidPts) + << "Expected >" << kMinValidPts << " valid points, got " + << valid_points.size(); + + // Collect z values and compute median. + std::vector z_vals; + z_vals.reserve(valid_points.size()); + for (const auto &pt : valid_points) { + z_vals.push_back(pt[2]); + } + + ASSERT_FALSE(z_vals.empty()) << "No valid points in cloud"; + + const auto mid = + z_vals.begin() + static_cast(z_vals.size() / 2); + std::nth_element(z_vals.begin(), mid, z_vals.end()); + const float median_z = *mid; + + // Report for the task summary. + std::cout << "[SCARED] valid_points=" << valid_points.size() + << " median_z=" << median_z << " m\n"; + + EXPECT_GE(median_z, kMinExpectedZ) + << "Median z " << median_z + << " m is below minimum expected " << kMinExpectedZ + << " m (check mm->m conversion)"; + EXPECT_LE(median_z, kMaxExpectedZ) + << "Median z " << median_z + << " m exceeds maximum expected " << kMaxExpectedZ + << " m (check mm->m conversion: T must be divided by 1000)"; +}