测试程序

/**
tcti.cpp

参考：
https://www.cnblogs.com/organic/p/17321523.html

g++ -std=c++11 -lpthread trigger_cgroup_timer_inactive.cpp -o inactive_timer
./inactive_timer 100000 10000
*/

#include <errno.h>
#include <iostream>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>

#ifndef CPU_CORE_COUNT
#define CPU_CORE_COUNT 64
#endif

using namespace std;
std::string sub_cgroup_dir("/sys/fs/cgroup/cpu/test");

// common lib
bool is_dir(const std::string &path) {
  struct stat statbuf;
  if (stat(path.c_str(), &statbuf) == 0) {
    if (0 != S_ISDIR(statbuf.st_mode)) {
      return true;
    }
  }
  return false;
}

bool write_file(const std::string &file_path, int num) {
  // std::cout << file_path << " op:" << num << std::endl;
  FILE *fp = fopen(file_path.c_str(), "w");
  if (fp == NULL) {
    return false;
  }

  // std::cout << file_path << " op:" << num << std::endl;
  std::string write_data = to_string(num);
  fputs(write_data.c_str(), fp);
  fclose(fp);
  return true;
}

std::string read_file(const std::string &file_path) {
  FILE *fp = fopen(file_path.c_str(), "r");
  if (NULL == fp) {
    return "read error...\n";
  }
  char buff[512];
  memset(buff, 0, 512);
  fread(buff, 512, 1, fp);
  fclose(fp);
  return std::string(buff);
}

// ms
long get_ms_timestamp() {
  timeval tv;
  gettimeofday(&tv, NULL);
  return (tv.tv_sec * 1000 + tv.tv_usec / 1000);
}

// cgroup
bool create_cgroup() {
  if (is_dir(sub_cgroup_dir) == false) {
    if (mkdir(sub_cgroup_dir.c_str(), S_IRWXU | S_IRGRP) != 0) {
      cout << "mkdir cgroup dir fail" << endl;
      return false;
    }
  }

  int pid = getpid();
  cout << "pid is " << pid << endl;
  std::string procs_path = sub_cgroup_dir + "/cgroup.procs";
  return write_file(procs_path, pid);
}

bool set_period(int period) {
  std::string period_path = sub_cgroup_dir + "/cpu.cfs_period_us";
  return write_file(period_path, period);
}

bool set_quota(int quota) {
  std::string quota_path = sub_cgroup_dir + "/cpu.cfs_quota_us";
  return write_file(quota_path, quota);
}

bool set_cpuOnline(int cpuId, int online) {
  std::string cpuPath = std::string("/sys/devices/system/cpu/cpu") +
                        std::to_string(cpuId) + std::string("/online");
  return write_file(cpuPath, online);
}

// thread
// param: ms interval
void *thread_func(void *param) {
  int i = 0;
  int interval = (long)param;
  long last = get_ms_timestamp();

  while (true) {
    i++;
    if (i % 100000 != 0) {
      continue;
    }

    long current = get_ms_timestamp();
    if ((current - last) >= interval) {
      last = current;
    }
  }

  pthread_exit(NULL);
}

void test_thread() {
  const int k_thread_num = CPU_CORE_COUNT * 10;
  pthread_t pthreads[k_thread_num];

  for (int i = 0; i < k_thread_num; i++) {
    if (pthread_create(&pthreads[i], NULL, thread_func, (void *)(i + 1)) != 0) {
      cout << "create thread fail" << endl;
    } else {
      cout << "create thread success,tid is " << pthreads[i] << endl;
    }
  }
}

void *thread_cpu_online_ctl(void *param) {
  int online, cpu;
  bool b;
  while (true) {
    online = rand() % 2;
    cpu = rand() % CPU_CORE_COUNT;
    b = set_cpuOnline(cpu, online);
    std::cout << cpu << " -> " << online << "result:" << b << std::endl;
    std::cout << read_file("/sys/devices/system/cpu/online") << std::endl;
  }

  pthread_exit(NULL);
}

void cpu_ctl_thread() {
  const int k_thread_num = CPU_CORE_COUNT;
  pthread_t pthreads[k_thread_num];

  for (int i = 0; i < k_thread_num; i++) {
    if (pthread_create(&pthreads[i], NULL, thread_cpu_online_ctl,
                       (void *)(i + 1)) != 0) {
      cout << "create thread fail" << endl;
    } else {
      cout << "create thread success,tid is " << pthreads[i] << endl;
    }
  }
}

int main(int argc, char *argv[]) {

  int period = 100000;
  int quota = CPU_CORE_COUNT * 0.8 * 100000;
  cout << "period is " << period << endl;
  cout << "quota is " << quota << endl;
  srand(time(nullptr));

  test_thread();
  cpu_ctl_thread();
  if (create_cgroup() == false) {
    cout << "create cgroup fail" << endl;
    return -1;
  }

  set_period(period);
  set_quota(quota);

  while (true) {
    sleep(10000);
  }

  return 0;
}

故障现象

64C64G运行一个多小时后，进程不再输出，无法停止（如果此时重启，系统会卡在重启过程）

crash 分析

virsh dump

virsh reset

vnc登录,crash

分析

ps | grep tcti |less

所有测试程序均标记为运行状态。实际上并没有CPU使用了了。

set pid ; runq -c cpuid

查看pid=2743进程状态为运行中，CPU号=2

runq查看CPU号=2的运行队列，显示no tasks queued

即：进程被标记在CPU2上正在运行，而CPU2上显示没有任何任务队列。

查看task对应的se没有在rq上，cfs_rq显示被throttled

runq -c 2 -g

由于CONFIG_RT_GROUP_SCHED未配置，因此无法查看task_group.rt_bandwith

参考以下文章

https://www.cnblogs.com/organic/p/17321523.html

根据其分析，当故障发生时，手动设置下cgroup限制即可触发定时器恢复调用

复测验证

复现进程假死

所有线程CPU均无使用率（ssh远程登录上去）

手动修改下cgroup配置

发现指令被卡主

说明本文的现象与

https://www.cnblogs.com/organic/p/17321523.html

所描述有差异。（同时也不可继续操作cpu online状态）

本文中测试程序添加了cpu online状态设置，原目的是尝试触发_nohz_idle_balance，分析以前发生的类似故障。

目前已知情况

OS	内核	故障
CentOS8.3.2011	4.18.0-240.el8.aarch64	容易复现
RedFlag7	4.19.133-300.axs7.14.aarch64	容易复现
openEuler 20.03 LTS SP3	4.19.90-2112.8.0.0131.oe1.aarch64	可复现
openEuler 22.03 LTS	5.10.0-60.18.0.50.oe2203.aarch64	未复现