18 m_slow_duration(std::chrono::seconds(1))
21 throw std::runtime_error(
"The storage statistics plugin must be chained with another filesystem.");
23 m_log.
Say(
"------ Initializing the storage statistics plugin.");
25 throw std::runtime_error(
"Failed to configure the storage statistics plugin.");
29 if ((rc =
XrdSysThread::Run(&tid, StatsFileSystem::AggregateBootstrap,
static_cast<void *
>(
this), 0,
"FS Stats Compute Thread"))) {
30 m_log.
Emsg(
"StatsFileSystem", rc,
"create stats compute thread");
31 throw std::runtime_error(
"Failed to create the statistics computing thread.");
35 m_log.
Say(
"Config",
"Stats monitoring has", m_gstream ?
"" :
" NOT",
" been configured via xrootd.mongstream directive");
42 StatsFileSystem::AggregateBootstrap(
void *me) {
45 std::this_thread::sleep_for(std::chrono::seconds(1));
46 myself->AggregateStats();
59 m_log.
Emsg(
"Config", -result,
"parsing config file", configfn);
66 if (!strcmp(val,
"trace")) {
69 m_log.
Emsg(
"Config",
"fsstats.trace requires an argument. Usage: fsstats.trace [all|err|warning|info|debug|none]");
78 else if (!strcmp(val,
"none")) {m_log.
setMsgMask(0);}
79 }
while ((val = statsConf.
GetToken()));
80 }
else if (!strcmp(val,
"slowop")) {
82 m_log.
Emsg(
"Config",
"fsstats.slowop requires an argument. Usage: fsstats.slowop [duration]");
87 m_log.
Emsg(
"Config",
"fsstats.slowop couldn't parse duration", val, errmsg.c_str());
100 std::unique_ptr<XrdOssDF> wrapped(m_oss->
newDir(user));
101 return new StatsDirectory(user, std::move(wrapped), m_log, *
this);
107 std::unique_ptr<XrdOssDF> wrapped(m_oss->
newFile(user));
108 return new StatsFile(std::move(wrapped), m_log, *
this);
113 OpTimer op(m_ops.m_chmod_ops, m_slow_ops.m_chmod_ops, m_times.m_chmod, m_slow_times.m_chmod, m_slow_duration);
114 return m_oss->
Chmod(path, mode, env);
125 return m_oss->
Create(tid, path, mode, env,
opts);
145 return m_oss->
FSctl(cmd, alen, args, resp);
155 return Init(lp, cfn);
161 return m_oss->
Mkdir(path, mode, mkpath, env);
165 const char *cgName,
const char *anchor)
178 OpTimer op(m_ops.m_rename_ops, m_slow_ops.m_rename_ops, m_times.m_rename, m_slow_times.m_rename, m_slow_duration);
179 return m_oss->
Rename(oPath, nPath, oEnvP, nEnvP);
185 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
186 return m_oss->
Stat(path, buff,
opts, env);
191 return m_oss->
Stats(buff, blen);
197 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
198 return m_oss->
StatFS(path, buff, blen, env);
202 char *buff,
int &blen)
204 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
205 return m_oss->
StatLS(env, path, buff, blen);
210 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
216 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
217 return m_oss->
StatPF(path, buff, 0);
222 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
223 return m_oss->
StatVS(vsP, sname, updt);
229 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
230 return m_oss->
StatXA(path, buff, blen, env);
236 OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
237 return m_oss->
StatXP(path, attr, env);
243 OpTimer op(m_ops.m_truncate_ops, m_slow_ops.m_truncate_ops, m_times.m_truncate, m_slow_times.m_truncate, m_slow_duration);
244 return m_oss->
Truncate(path, fsize, env);
249 OpTimer op(m_ops.m_unlink_ops, m_slow_ops.m_unlink_ops, m_times.m_unlink, m_slow_times.m_unlink, m_slow_duration);
263 void StatsFileSystem::AggregateStats()
266 auto len = snprintf(buf, 1500,
268 "\"event\":\"oss_stats\"," \
269 "\"reads\":%" PRIu64
",\"writes\":%" PRIu64
",\"stats\":%" PRIu64
"," \
270 "\"pgreads\":%" PRIu64
",\"pgwrites\":%" PRIu64
",\"readvs\":%" PRIu64
"," \
271 "\"readv_segs\":%" PRIu64
",\"dirlists\":%" PRIu64
",\"dirlist_ents\":%" PRIu64
","
272 "\"truncates\":%" PRIu64
",\"unlinks\":%" PRIu64
",\"chmods\":%" PRIu64
","
273 "\"opens\":%" PRIu64
",\"renames\":%" PRIu64
","
274 "\"slow_reads\":%" PRIu64
",\"slow_writes\":%" PRIu64
",\"slow_stats\":%" PRIu64
","
275 "\"slow_pgreads\":%" PRIu64
",\"slow_pgwrites\":%" PRIu64
",\"slow_readvs\":%" PRIu64
","
276 "\"slow_readv_segs\":%" PRIu64
",\"slow_dirlists\":%" PRIu64
",\"slow_dirlist_ents\":%" PRIu64
","
277 "\"slow_truncates\":%" PRIu64
",\"slow_unlinks\":%" PRIu64
",\"slow_chmods\":%" PRIu64
","
278 "\"slow_opens\":%" PRIu64
",\"slow_renames\":%" PRIu64
","
279 "\"open_t\":%.4f,\"read_t\":%.4f,\"readv_t\":%.4f,"
280 "\"pgread_t\":%.4f,\"write_t\":%.4f,\"pgwrite_t\":%.4f,"
281 "\"dirlist_t\":%.4f,\"stat_t\":%.4f,\"truncate_t\":%.4f,"
282 "\"unlink_t\":%.4f,\"rename_t\":%.4f,\"chmod_t\":%.4f,"
283 "\"slow_open_t\":%.4f,\"slow_read_t\":%.4f,\"slow_readv_t\":%.4f,"
284 "\"slow_pgread_t\":%.4f,\"slow_write_t\":%.4f,\"slow_pgwrite_t\":%.4f,"
285 "\"slow_dirlist_t\":%.4f,\"slow_stat_t\":%.4f,\"slow_truncate_t\":%.4f,"
286 "\"slow_unlink_t\":%.4f,\"slow_rename_t\":%.4f,\"slow_chmod_t\":%.4f"
288 m_ops.m_read_ops.load(), m_ops.m_write_ops.load(), m_ops.m_stat_ops.load(),
289 m_ops.m_pgread_ops.load(), m_ops.m_pgwrite_ops.load(), m_ops.m_readv_ops.load(),
290 m_ops.m_readv_segs.load(), m_ops.m_dirlist_ops.load(), m_ops.m_dirlist_entries.load(),
291 m_ops.m_truncate_ops.load(), m_ops.m_unlink_ops.load(), m_ops.m_chmod_ops.load(),
292 m_ops.m_open_ops.load(), m_ops.m_rename_ops.load(),
293 m_slow_ops.m_read_ops.load(), m_slow_ops.m_write_ops.load(), m_slow_ops.m_stat_ops.load(),
294 m_slow_ops.m_pgread_ops.load(), m_slow_ops.m_pgwrite_ops.load(), m_slow_ops.m_readv_ops.load(),
295 m_slow_ops.m_readv_segs.load(), m_slow_ops.m_dirlist_ops.load(), m_slow_ops.m_dirlist_entries.load(),
296 m_slow_ops.m_truncate_ops.load(), m_slow_ops.m_unlink_ops.load(), m_slow_ops.m_chmod_ops.load(),
297 m_slow_ops.m_open_ops.load(), m_slow_ops.m_rename_ops.load(),
298 static_cast<float>(m_times.m_open.load())/1e9,
static_cast<float>(m_times.m_read.load())/1e9,
static_cast<float>(m_times.m_readv.load())/1e9,
299 static_cast<float>(m_times.m_pgread.load())/1e9,
static_cast<float>(m_times.m_write.load())/1e9,
static_cast<float>(m_times.m_pgwrite.load())/1e9,
300 static_cast<float>(m_times.m_dirlist.load())/1e9,
static_cast<float>(m_times.m_stat.load())/1e9,
static_cast<float>(m_times.m_truncate.load())/1e9,
301 static_cast<float>(m_times.m_unlink.load())/1e9,
static_cast<float>(m_times.m_rename.load())/1e9,
static_cast<float>(m_times.m_chmod.load())/1e9,
302 static_cast<float>(m_slow_times.m_open.load())/1e9,
static_cast<float>(m_slow_times.m_read.load())/1e9,
static_cast<float>(m_slow_times.m_readv.load())/1e9,
303 static_cast<float>(m_slow_times.m_pgread.load())/1e9,
static_cast<float>(m_slow_times.m_write.load())/1e9,
static_cast<float>(m_slow_times.m_pgwrite.load())/1e9,
304 static_cast<float>(m_slow_times.m_dirlist.load())/1e9,
static_cast<float>(m_slow_times.m_stat.load())/1e9,
static_cast<float>(m_slow_times.m_truncate.load())/1e9,
305 static_cast<float>(m_slow_times.m_unlink.load())/1e9,
static_cast<float>(m_slow_times.m_rename.load())/1e9,
static_cast<float>(m_slow_times.m_chmod.load())/1e9
309 m_log.
Log(
LogMask::Error,
"Aggregate",
"Failed to generate g-stream statistics packet");
313 if (m_gstream && !m_gstream->
Insert(buf, len + 1)) {
314 m_log.
Log(
LogMask::Error,
"Aggregate",
"Failed to send g-stream statistics packet");
319 StatsFileSystem::OpTimer::OpTimer(std::atomic<uint64_t> &op_count, std::atomic<uint64_t> &slow_op_count, std::atomic<uint64_t> &timing, std::atomic<uint64_t> &slow_timing, std::chrono::steady_clock::duration duration)
320 : m_op_count(op_count),
321 m_slow_op_count(slow_op_count),
323 m_slow_timing(slow_timing),
324 m_start(std::chrono::steady_clock::now()),
325 m_slow_duration(duration)
328 StatsFileSystem::OpTimer::~OpTimer()
330 auto dur = std::chrono::steady_clock::now() - m_start;
332 m_timing += std::chrono::nanoseconds(dur).count();
333 if (dur > m_slow_duration) {
335 m_slow_timing += std::chrono::nanoseconds(dur).count();
int stat(const char *path, struct stat *buf)
bool ParseDuration(const std::string &duration, std::chrono::steady_clock::duration &result, std::string &errmsg)
std::string LogMaskToString(int mask)
void Disc(XrdOucEnv &env) override
int Init(XrdSysLogger *lp, const char *cfn) override
XrdOssDF * newDir(const char *user=0) override
StatsFileSystem(XrdOss *oss, XrdSysLogger *log, const char *configName, XrdOucEnv *envP)
void EnvInfo(XrdOucEnv *env) override
void Connect(XrdOucEnv &env) override
int Stat(const char *path, struct stat *buff, int opts=0, XrdOucEnv *env=0) override
int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *env=0) override
virtual ~StatsFileSystem()
int Reloc(const char *tident, const char *path, const char *cgName, const char *anchor=0) override
int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *env=0) override
int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, int opts=0) override
XrdOssDF * newFile(const char *user=0) override
friend class StatsDirectory
bool Config(const char *configfn)
int StatPF(const char *path, struct stat *buff, int opts) override
int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen) override
int Stats(char *buff, int blen) override
int Remdir(const char *path, int Opts=0, XrdOucEnv *env=0) override
int Unlink(const char *path, int Opts=0, XrdOucEnv *env=0) override
int Mkdir(const char *path, mode_t mode, int mkpath=0, XrdOucEnv *env=0) override
int FSctl(int cmd, int alen, const char *args, char **resp=0) override
uint64_t Features() override
int Chmod(const char *path, mode_t mode, XrdOucEnv *env=0) override
int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0) override
int Truncate(const char *path, unsigned long long fsize, XrdOucEnv *env=0) override
int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *env=0) override
int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0) override
int Lfn2Pfn(const char *Path, char *buff, int blen) override
virtual int Mkdir(const char *path, mode_t mode, int mkpath=0, XrdOucEnv *envP=0)=0
virtual int FSctl(int cmd, int alen, const char *args, char **resp=0)
virtual int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen)
virtual int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *envP=0)
virtual int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, int opts=0)=0
virtual int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *envP=0)
virtual XrdOssDF * newDir(const char *tident)=0
virtual void Connect(XrdOucEnv &env)
virtual int Reloc(const char *tident, const char *path, const char *cgName, const char *anchor=0)
virtual int Chmod(const char *path, mode_t mode, XrdOucEnv *envP=0)=0
virtual uint64_t Features()
virtual int StatPF(const char *path, struct stat *buff, int opts)
virtual void Disc(XrdOucEnv &env)
virtual int Lfn2Pfn(const char *Path, char *buff, int blen)
virtual int Remdir(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
virtual int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *envP=0)
virtual void EnvInfo(XrdOucEnv *envP)
virtual int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0)=0
virtual XrdOssDF * newFile(const char *tident)=0
virtual int Truncate(const char *path, unsigned long long fsize, XrdOucEnv *envP=0)=0
virtual int Stats(char *buff, int blen)
virtual int Stat(const char *path, struct stat *buff, int opts=0, XrdOucEnv *envP=0)=0
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
void * GetPtr(const char *varname)
char * GetToken(char **rest=0, int lowcase=0)
int Gather(const char *cfname, Level lvl, const char *parms=0)
@ trim_lines
Prefix trimmed lines.
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
void setMsgMask(int mask)
void Log(int mask, const char *esfx, const char *text1, const char *text2=0, const char *text3=0)
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)
bool Insert(const char *data, int dlen)