cmvr-es/cmvr-es/manager/task_manager/tests/task_manager_lifecycle_test.cpp

#include "manager/task_manager/include/task_manager.h"

#include <atomic>
#include <condition_variable>
#include <future>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>

#include <gtest/gtest.h>

#include "service/stop_all/include/stop_all_admission_gate.h"
#include "task/task_factory.h"

namespace {

struct TaskBehavior {
    bool init_result{true};
    bool start_result{true};
    bool throw_on_start{false};
    bool stop_activity_result{true};
    bool block_start{false};
    bool query_snapshot_on_stop{false};
};

TaskBehavior task_behavior;
std::vector<std::string> stop_order;

class LifecycleTask final : public cmvr::task::Task {
public:
    explicit LifecycleTask(
        std::string id,
        const cmvr::task::TaskShutdownPhase shutdown_phase =
            cmvr::task::TaskShutdownPhase::DEPENDENT_ACTIVITY)
        : id_(std::move(id)),
          shutdown_phase_(shutdown_phase)
    {
    }

    const std::string& id() const override { return id_; }
    cmvr::task::TaskRunMode runMode() const override
    {
        return cmvr::task::TaskRunMode::BLOCKING_SERVICE;
    }
    cmvr::task::TaskShutdownPhase shutdownPhase() const override
    {
        return shutdown_phase_;
    }

    bool init() override
    {
        ++init_calls;
        state_ = task_behavior.init_result
                     ? cmvr::task::TaskState::IDLE
                     : cmvr::task::TaskState::FAILED;
        return task_behavior.init_result;
    }

    bool start() override
    {
        ++start_calls;
        {
            std::unique_lock lock(start_mutex);
            start_entered = true;
            start_condition.notify_all();
            start_condition.wait(lock, [] {
                return !task_behavior.block_start;
            });
        }
        if (task_behavior.throw_on_start) {
            throw std::runtime_error("start failure");
        }
        state_ = task_behavior.start_result
                     ? cmvr::task::TaskState::RUNNING
                     : cmvr::task::TaskState::FAILED;
        return task_behavior.start_result;
    }

    bool step(double) override { return true; }

    void stop() override
    {
        ++stop_calls;
        if (task_behavior.query_snapshot_on_stop) {
            (void)cmvr::task::TaskManager::activitySnapshotIfInitialized();
        }
        stop_order.push_back(id_);
        state_ = cmvr::task::TaskState::STOPPED;
    }

    bool stopActivity() override
    {
        ++stop_activity_calls;
        return task_behavior.stop_activity_result;
    }

    cmvr::task::TaskState state() const override { return state_; }
    bool isBusy() const override
    {
        return state_ == cmvr::task::TaskState::RUNNING;
    }
    bool isFinished() const override
    {
        return state_ == cmvr::task::TaskState::STOPPED;
    }
    bool isFailed() const override
    {
        return state_ == cmvr::task::TaskState::FAILED;
    }
    std::string stateString() const override
    {
        return cmvr::task::taskStateToString(state_);
    }
    std::string detailStatusString() const override
    {
        return stateString();
    }

    int init_calls{0};
    int start_calls{0};
    int stop_calls{0};
    int stop_activity_calls{0};
    std::mutex start_mutex;
    std::condition_variable start_condition;
    bool start_entered{false};

private:
    std::string id_;
    cmvr::task::TaskShutdownPhase shutdown_phase_;
    cmvr::task::TaskState state_{
        cmvr::task::TaskState::UNINITIALIZED};
};

std::shared_ptr<LifecycleTask> created_task;
std::shared_ptr<LifecycleTask> created_ingress_task;

cmvr::config::TaskManagerConfig enabledTaskConfig()
{
    cmvr::config::TaskManagerConfig config;
    auto* entry = config.add_tasks();
    entry->set_id("lifecycle_task");
    entry->set_type(
        cmvr::config::TaskConfigEntry::TASK_TYPE_UME_TELEOP);
    entry->set_enable(true);
    entry->set_run_mode(
        cmvr::config::TaskConfigEntry::
            TASK_RUN_MODE_BLOCKING_SERVICE);
    return config;
}

class TaskManagerLifecycleTest : public ::testing::Test {
protected:
    void SetUp() override
    {
        cmvr::task::TaskManager::destroyInstance();
        cmvr::service::globalStopAllAdmissionGate().clearForTesting();
        task_behavior = {};
        created_task.reset();
        created_ingress_task.reset();
        stop_order.clear();
        cmvr::task::TaskFactory::registerCreator(
            cmvr::config::TaskConfigEntry::TASK_TYPE_UME_TELEOP,
            [](const cmvr::config::TaskConfigEntry& entry) {
                created_task =
                    std::make_shared<LifecycleTask>(entry.id());
                return created_task;
            });
    }

    void TearDown() override
    {
        if (created_task) {
            {
                std::lock_guard lock(created_task->start_mutex);
                task_behavior.block_start = false;
            }
            created_task->start_condition.notify_all();
        }
        cmvr::task::TaskManager::destroyInstance();
        cmvr::service::globalStopAllAdmissionGate().clearForTesting();
        created_task.reset();
        created_ingress_task.reset();
        stop_order.clear();
    }
};

TEST_F(TaskManagerLifecycleTest,
       EnabledTaskInitFailureMarksInitializationFailed)
{
    task_behavior.init_result = false;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    ASSERT_NE(created_task, nullptr);
    EXPECT_FALSE(manager.initialized());
    EXPECT_FALSE(manager.startRunTask());
    EXPECT_FALSE(manager.running());
    EXPECT_EQ(created_task->init_calls, 1);
    EXPECT_EQ(created_task->start_calls, 0);
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest,
       TaskStartFailureIsReturnedAndRunningRemainsFalse)
{
    task_behavior.start_result = false;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    ASSERT_TRUE(manager.initialized());
    ASSERT_NE(created_task, nullptr);
    EXPECT_FALSE(manager.startRunTask());
    EXPECT_FALSE(manager.running());
    EXPECT_EQ(created_task->start_calls, 1);
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest,
       TaskStartExceptionIsReturnedAndRunningRemainsFalse)
{
    task_behavior.throw_on_start = true;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    ASSERT_TRUE(manager.initialized());
    ASSERT_NE(created_task, nullptr);
    EXPECT_FALSE(manager.startRunTask());
    EXPECT_FALSE(manager.running());
    EXPECT_EQ(created_task->start_calls, 1);
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest, SuccessfulStartAndStopAreReported)
{
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    ASSERT_TRUE(manager.initialized());
    ASSERT_NE(created_task, nullptr);
    EXPECT_TRUE(manager.startRunTask());
    EXPECT_TRUE(manager.running());
    manager.stopRunTask();
    EXPECT_FALSE(manager.running());
    EXPECT_EQ(created_task->start_calls, 1);
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest,
       CommandIngressStopsBeforeDependentTaskActivity)
{
    cmvr::task::TaskFactory::registerCreator(
        cmvr::config::TaskConfigEntry::TASK_TYPE_GRPC_SERVER,
        [](const cmvr::config::TaskConfigEntry& entry) {
            created_ingress_task = std::make_shared<LifecycleTask>(
                entry.id(),
                cmvr::task::TaskShutdownPhase::COMMAND_INGRESS);
            return created_ingress_task;
        });

    auto config = enabledTaskConfig();
    auto* ingress = config.add_tasks();
    ingress->set_id("control_ingress");
    ingress->set_type(
        cmvr::config::TaskConfigEntry::TASK_TYPE_GRPC_SERVER);
    ingress->set_enable(true);
    ingress->set_run_mode(
        cmvr::config::TaskConfigEntry::TASK_RUN_MODE_BLOCKING_SERVICE);

    auto& manager = cmvr::task::TaskManager::getInstance(config);
    ASSERT_TRUE(manager.initialized());
    ASSERT_TRUE(manager.startRunTask());
    manager.stopRunTask();

    ASSERT_EQ(stop_order.size(), 2U);
    EXPECT_EQ(stop_order[0], "control_ingress");
    EXPECT_EQ(stop_order[1], "lifecycle_task");
}

TEST_F(TaskManagerLifecycleTest,
       DestroyDoesNotHoldSingletonLockWhileStoppingTasks)
{
    task_behavior.query_snapshot_on_stop = true;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());
    ASSERT_TRUE(manager.initialized());
    ASSERT_TRUE(manager.startRunTask());

    auto destroy = std::async(std::launch::async, [] {
        cmvr::task::TaskManager::destroyInstance();
    });
    ASSERT_EQ(
        destroy.wait_for(std::chrono::seconds(1)),
        std::future_status::ready);
    destroy.get();

    ASSERT_NE(created_task, nullptr);
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest,
       StopAllActivitiesDoesNotStopSchedulerOrTaskLifecycle)
{
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    ASSERT_TRUE(manager.initialized());
    ASSERT_TRUE(manager.startRunTask());
    ASSERT_NE(created_task, nullptr);
    EXPECT_TRUE(manager.stopAllActivities());
    EXPECT_TRUE(manager.running());
    EXPECT_EQ(created_task->stop_activity_calls, 1);
    EXPECT_EQ(created_task->stop_calls, 0);

    manager.stopRunTask();
    EXPECT_EQ(created_task->stop_calls, 1);
}

TEST_F(TaskManagerLifecycleTest,
       StopAllActivitiesReportsUnconfirmedOperationalStop)
{
    task_behavior.stop_activity_result = false;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());

    std::vector<std::string> failures;
    EXPECT_FALSE(manager.stopAllActivities(&failures));
    ASSERT_EQ(failures.size(), 1U);
    EXPECT_NE(failures.front().find("lifecycle_task"), std::string::npos);
    EXPECT_EQ(created_task->stop_calls, 0);
}

TEST_F(TaskManagerLifecycleTest,
       StopAllActivitiesIsSuccessfulWhenManagerIsNotInitialized)
{
    cmvr::task::TaskManager::destroyInstance();
    std::vector<std::string> failures;
    EXPECT_TRUE(
        cmvr::task::TaskManager::stopAllActivitiesIfInitialized(&failures));
    EXPECT_TRUE(failures.empty());
}

TEST_F(TaskManagerLifecycleTest, StartIsRejectedWhileStopAllAdmissionIsClosed)
{
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());
    auto& gate = cmvr::service::globalStopAllAdmissionGate();
    const auto ticket = gate.beginStopAll();

    EXPECT_FALSE(manager.startRunTask());
    EXPECT_FALSE(manager.running());
    ASSERT_NE(created_task, nullptr);
    EXPECT_EQ(created_task->start_calls, 0);

    EXPECT_TRUE(gate.finishStopAll(ticket, true));
    EXPECT_TRUE(manager.startRunTask());
    EXPECT_TRUE(manager.running());
}

TEST_F(TaskManagerLifecycleTest,
       StartCrossingStopAllGenerationRollsBackStartedTasks)
{
    task_behavior.block_start = true;
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());
    ASSERT_NE(created_task, nullptr);

    std::atomic<bool> start_result{true};
    std::thread starter([&] {
        start_result.store(manager.startRunTask());
    });
    bool start_entered = false;
    {
        std::unique_lock lock(created_task->start_mutex);
        start_entered = created_task->start_condition.wait_for(
            lock, std::chrono::seconds(2), [&] {
                return created_task->start_entered;
            });
    }
    if (!start_entered) {
        {
            std::lock_guard lock(created_task->start_mutex);
            task_behavior.block_start = false;
        }
        created_task->start_condition.notify_all();
        starter.join();
        FAIL() << "task start did not reach the generation-race barrier";
    }

    auto& gate = cmvr::service::globalStopAllAdmissionGate();
    const auto ticket = gate.beginStopAll();
    {
        std::lock_guard lock(created_task->start_mutex);
        task_behavior.block_start = false;
    }
    created_task->start_condition.notify_all();
    starter.join();

    EXPECT_FALSE(start_result.load());
    EXPECT_FALSE(manager.running());
    EXPECT_EQ(created_task->start_calls, 1);
    EXPECT_EQ(created_task->stop_calls, 1);

    EXPECT_TRUE(gate.finishStopAll(ticket, true));
    EXPECT_TRUE(manager.startRunTask());
}

TEST_F(TaskManagerLifecycleTest,
       FailedStopAllKeepsTaskStartupRejectedUntilSuccessfulRound)
{
    auto& manager =
        cmvr::task::TaskManager::getInstance(enabledTaskConfig());
    auto& gate = cmvr::service::globalStopAllAdmissionGate();
    auto ticket = gate.beginStopAll();
    EXPECT_FALSE(gate.finishStopAll(ticket, false));

    EXPECT_FALSE(manager.startRunTask());
    ASSERT_NE(created_task, nullptr);
    EXPECT_EQ(created_task->start_calls, 0);

    ticket = gate.beginStopAll();
    EXPECT_TRUE(gate.finishStopAll(ticket, true));
    EXPECT_TRUE(manager.startRunTask());
}

} // namespace