XRootD
XrdStatsFileSystem.cc
Go to the documentation of this file.
1 
3 #include "XrdStatsConfig.hh"
4 #include "XrdStatsDirectory.hh"
5 #include "XrdStatsFile.hh"
6 #include "XrdStatsFileSystem.hh"
9 
10 #include <inttypes.h>
11 #include <stdexcept>
12 #include <thread>
13 
15  m_oss(oss),
16  m_env(envP),
17  m_log(lp, "fsstat_"),
18  m_slow_duration(std::chrono::seconds(1))
19 {
20  if (!oss) {
21  throw std::runtime_error("The storage statistics plugin must be chained with another filesystem.");
22  }
23  m_log.Say("------ Initializing the storage statistics plugin.");
24  if (!Config(configfn)) {
25  throw std::runtime_error("Failed to configure the storage statistics plugin.");
26  }
27  pthread_t tid;
28  int rc;
29  if ((rc = XrdSysThread::Run(&tid, StatsFileSystem::AggregateBootstrap, static_cast<void *>(this), 0, "FS Stats Compute Thread"))) {
30  m_log.Emsg("StatsFileSystem", rc, "create stats compute thread");
31  throw std::runtime_error("Failed to create the statistics computing thread.");
32  }
33  if (envP) {
34  m_gstream = reinterpret_cast<XrdXrootdGStream*>(envP->GetPtr("oss.gStream*"));
35  m_log.Say("Config", "Stats monitoring has", m_gstream ? "" : " NOT", " been configured via xrootd.mongstream directive");
36  }
37 }
38 
40 
41 void *
42 StatsFileSystem::AggregateBootstrap(void *me) {
43  auto myself = static_cast<StatsFileSystem*>(me);
44  while (1) {
45  std::this_thread::sleep_for(std::chrono::seconds(1));
46  myself->AggregateStats();
47  }
48  return nullptr;
49 }
50 
51 bool
52 StatsFileSystem::Config(const char *configfn)
53 {
55 
56  XrdOucGatherConf statsConf("fsstats.trace fsstats.slowop", &m_log);
57  int result;
58  if ((result = statsConf.Gather(configfn, XrdOucGatherConf::trim_lines)) < 0) {
59  m_log.Emsg("Config", -result, "parsing config file", configfn);
60  return false;
61  }
62 
63  char *val;
64  while (statsConf.GetLine()) {
65  val = statsConf.GetToken(); // Ignore -- we asked for a single value
66  if (!strcmp(val, "trace")) {
67  m_log.setMsgMask(0);
68  if (!(val = statsConf.GetToken())) {
69  m_log.Emsg("Config", "fsstats.trace requires an argument. Usage: fsstats.trace [all|err|warning|info|debug|none]");
70  return false;
71  }
72  do {
73  if (!strcmp(val, "all")) {m_log.setMsgMask(m_log.getMsgMask() | LogMask::All);}
74  else if (!strcmp(val, "error")) {m_log.setMsgMask(m_log.getMsgMask() | LogMask::Error);}
75  else if (!strcmp(val, "warning")) {m_log.setMsgMask(m_log.getMsgMask() | LogMask::Error | LogMask::Warning);}
76  else if (!strcmp(val, "info")) {m_log.setMsgMask(m_log.getMsgMask() | LogMask::Error | LogMask::Warning | LogMask::Info);}
77  else if (!strcmp(val, "debug")) {m_log.setMsgMask(m_log.getMsgMask() | LogMask::Error | LogMask::Warning | LogMask::Info | LogMask::Debug);}
78  else if (!strcmp(val, "none")) {m_log.setMsgMask(0);}
79  } while ((val = statsConf.GetToken()));
80  } else if (!strcmp(val, "slowop")) {
81  if (!(val = statsConf.GetToken())) {
82  m_log.Emsg("Config", "fsstats.slowop requires an argument. Usage: fsstats.slowop [duration]");
83  return false;
84  }
85  std::string errmsg;
86  if (!ParseDuration(val, m_slow_duration, errmsg)) {
87  m_log.Emsg("Config", "fsstats.slowop couldn't parse duration", val, errmsg.c_str());
88  return false;
89  }
90  }
91  }
92  m_log.Emsg("Config", "Logging levels enabled", LogMaskToString(m_log.getMsgMask()).c_str());
93 
94  return true;
95 }
96 
98 {
99  // Call the underlying OSS newDir
100  std::unique_ptr<XrdOssDF> wrapped(m_oss->newDir(user));
101  return new StatsDirectory(user, std::move(wrapped), m_log, *this);
102 }
103 
105 {
106  // Call the underlying OSS newFile
107  std::unique_ptr<XrdOssDF> wrapped(m_oss->newFile(user));
108  return new StatsFile(std::move(wrapped), m_log, *this);
109 }
110 
111 int StatsFileSystem::Chmod(const char * path, mode_t mode, XrdOucEnv *env)
112 {
113  OpTimer op(m_ops.m_chmod_ops, m_slow_ops.m_chmod_ops, m_times.m_chmod, m_slow_times.m_chmod, m_slow_duration);
114  return m_oss->Chmod(path, mode, env);
115 }
116 
118 {
119  m_oss->Connect(env);
120 }
121 
122 int StatsFileSystem::Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env,
123  int opts)
124 {
125  return m_oss->Create(tid, path, mode, env, opts);
126 }
127 
129 {
130  m_oss->Disc(env);
131 }
132 
134 {
135  m_oss->EnvInfo(env);
136 }
137 
139 {
140  return m_oss->Features();
141 }
142 
143 int StatsFileSystem::FSctl(int cmd, int alen, const char *args, char **resp)
144 {
145  return m_oss->FSctl(cmd, alen, args, resp);
146 }
147 
148 int StatsFileSystem::Init(XrdSysLogger *lp, const char *cfn)
149 {
150  return 0;
151 }
152 
153 int StatsFileSystem::Init(XrdSysLogger *lp, const char *cfn, XrdOucEnv *env)
154 {
155  return Init(lp, cfn);
156 }
157 
158 int StatsFileSystem::Mkdir(const char *path, mode_t mode, int mkpath,
159  XrdOucEnv *env)
160 {
161  return m_oss->Mkdir(path, mode, mkpath, env);
162 }
163 
164 int StatsFileSystem::Reloc(const char *tident, const char *path,
165  const char *cgName, const char *anchor)
166 {
167  return m_oss->Reloc(tident, path, cgName, anchor);
168 }
169 
170 int StatsFileSystem::Remdir(const char *path, int Opts, XrdOucEnv *env)
171 {
172  return m_oss->Remdir(path, Opts, env);
173 }
174 
175 int StatsFileSystem::Rename(const char *oPath, const char *nPath,
176  XrdOucEnv *oEnvP, XrdOucEnv *nEnvP)
177 {
178  OpTimer op(m_ops.m_rename_ops, m_slow_ops.m_rename_ops, m_times.m_rename, m_slow_times.m_rename, m_slow_duration);
179  return m_oss->Rename(oPath, nPath, oEnvP, nEnvP);
180 }
181 
182 int StatsFileSystem::Stat(const char *path, struct stat *buff,
183  int opts, XrdOucEnv *env)
184 {
185  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
186  return m_oss->Stat(path, buff, opts, env);
187 }
188 
189 int StatsFileSystem::Stats(char *buff, int blen)
190 {
191  return m_oss->Stats(buff, blen);
192 }
193 
194 int StatsFileSystem::StatFS(const char *path, char *buff, int &blen,
195  XrdOucEnv *env)
196 {
197  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
198  return m_oss->StatFS(path, buff, blen, env);
199 }
200 
201 int StatsFileSystem::StatLS(XrdOucEnv &env, const char *path,
202  char *buff, int &blen)
203 {
204  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
205  return m_oss->StatLS(env, path, buff, blen);
206 }
207 
208 int StatsFileSystem::StatPF(const char *path, struct stat *buff, int opts)
209 {
210  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
211  return m_oss->StatPF(path, buff, opts);
212 }
213 
214 int StatsFileSystem::StatPF(const char *path, struct stat *buff)
215 {
216  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
217  return m_oss->StatPF(path, buff, 0);
218 }
219 
220 int StatsFileSystem::StatVS(XrdOssVSInfo *vsP, const char *sname, int updt)
221 {
222  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
223  return m_oss->StatVS(vsP, sname, updt);
224 }
225 
226 int StatsFileSystem::StatXA(const char *path, char *buff, int &blen,
227  XrdOucEnv *env)
228 {
229  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
230  return m_oss->StatXA(path, buff, blen, env);
231 }
232 
233 int StatsFileSystem::StatXP(const char *path, unsigned long long &attr,
234  XrdOucEnv *env)
235 {
236  OpTimer op(m_ops.m_stat_ops, m_slow_ops.m_stat_ops, m_times.m_stat, m_slow_times.m_stat, m_slow_duration);
237  return m_oss->StatXP(path, attr, env);
238 }
239 
240 int StatsFileSystem::Truncate(const char *path, unsigned long long fsize,
241  XrdOucEnv *env)
242 {
243  OpTimer op(m_ops.m_truncate_ops, m_slow_ops.m_truncate_ops, m_times.m_truncate, m_slow_times.m_truncate, m_slow_duration);
244  return m_oss->Truncate(path, fsize, env);
245 }
246 
247 int StatsFileSystem::Unlink(const char *path, int Opts, XrdOucEnv *env)
248 {
249  OpTimer op(m_ops.m_unlink_ops, m_slow_ops.m_unlink_ops, m_times.m_unlink, m_slow_times.m_unlink, m_slow_duration);
250  return m_oss->Unlink(path, Opts, env);
251 }
252 
253 int StatsFileSystem::Lfn2Pfn(const char *Path, char *buff, int blen)
254 {
255  return m_oss->Lfn2Pfn(Path, buff, blen);
256 }
257 
258 const char *StatsFileSystem::Lfn2Pfn(const char *Path, char *buff, int blen, int &rc)
259 {
260  return m_oss->Lfn2Pfn(Path, buff, blen, rc);
261 }
262 
263 void StatsFileSystem::AggregateStats()
264 {
265  char buf[1500];
266  auto len = snprintf(buf, 1500,
267  "{"
268  "\"event\":\"oss_stats\"," \
269  "\"reads\":%" PRIu64 ",\"writes\":%" PRIu64 ",\"stats\":%" PRIu64 "," \
270  "\"pgreads\":%" PRIu64 ",\"pgwrites\":%" PRIu64 ",\"readvs\":%" PRIu64 "," \
271  "\"readv_segs\":%" PRIu64 ",\"dirlists\":%" PRIu64 ",\"dirlist_ents\":%" PRIu64 ","
272  "\"truncates\":%" PRIu64 ",\"unlinks\":%" PRIu64 ",\"chmods\":%" PRIu64 ","
273  "\"opens\":%" PRIu64 ",\"renames\":%" PRIu64 ","
274  "\"slow_reads\":%" PRIu64 ",\"slow_writes\":%" PRIu64 ",\"slow_stats\":%" PRIu64 ","
275  "\"slow_pgreads\":%" PRIu64 ",\"slow_pgwrites\":%" PRIu64 ",\"slow_readvs\":%" PRIu64 ","
276  "\"slow_readv_segs\":%" PRIu64 ",\"slow_dirlists\":%" PRIu64 ",\"slow_dirlist_ents\":%" PRIu64 ","
277  "\"slow_truncates\":%" PRIu64 ",\"slow_unlinks\":%" PRIu64 ",\"slow_chmods\":%" PRIu64 ","
278  "\"slow_opens\":%" PRIu64 ",\"slow_renames\":%" PRIu64 ","
279  "\"open_t\":%.4f,\"read_t\":%.4f,\"readv_t\":%.4f,"
280  "\"pgread_t\":%.4f,\"write_t\":%.4f,\"pgwrite_t\":%.4f,"
281  "\"dirlist_t\":%.4f,\"stat_t\":%.4f,\"truncate_t\":%.4f,"
282  "\"unlink_t\":%.4f,\"rename_t\":%.4f,\"chmod_t\":%.4f,"
283  "\"slow_open_t\":%.4f,\"slow_read_t\":%.4f,\"slow_readv_t\":%.4f,"
284  "\"slow_pgread_t\":%.4f,\"slow_write_t\":%.4f,\"slow_pgwrite_t\":%.4f,"
285  "\"slow_dirlist_t\":%.4f,\"slow_stat_t\":%.4f,\"slow_truncate_t\":%.4f,"
286  "\"slow_unlink_t\":%.4f,\"slow_rename_t\":%.4f,\"slow_chmod_t\":%.4f"
287  "}",
288  m_ops.m_read_ops.load(), m_ops.m_write_ops.load(), m_ops.m_stat_ops.load(),
289  m_ops.m_pgread_ops.load(), m_ops.m_pgwrite_ops.load(), m_ops.m_readv_ops.load(),
290  m_ops.m_readv_segs.load(), m_ops.m_dirlist_ops.load(), m_ops.m_dirlist_entries.load(),
291  m_ops.m_truncate_ops.load(), m_ops.m_unlink_ops.load(), m_ops.m_chmod_ops.load(),
292  m_ops.m_open_ops.load(), m_ops.m_rename_ops.load(),
293  m_slow_ops.m_read_ops.load(), m_slow_ops.m_write_ops.load(), m_slow_ops.m_stat_ops.load(),
294  m_slow_ops.m_pgread_ops.load(), m_slow_ops.m_pgwrite_ops.load(), m_slow_ops.m_readv_ops.load(),
295  m_slow_ops.m_readv_segs.load(), m_slow_ops.m_dirlist_ops.load(), m_slow_ops.m_dirlist_entries.load(),
296  m_slow_ops.m_truncate_ops.load(), m_slow_ops.m_unlink_ops.load(), m_slow_ops.m_chmod_ops.load(),
297  m_slow_ops.m_open_ops.load(), m_slow_ops.m_rename_ops.load(),
298  static_cast<float>(m_times.m_open.load())/1e9, static_cast<float>(m_times.m_read.load())/1e9, static_cast<float>(m_times.m_readv.load())/1e9,
299  static_cast<float>(m_times.m_pgread.load())/1e9, static_cast<float>(m_times.m_write.load())/1e9, static_cast<float>(m_times.m_pgwrite.load())/1e9,
300  static_cast<float>(m_times.m_dirlist.load())/1e9, static_cast<float>(m_times.m_stat.load())/1e9, static_cast<float>(m_times.m_truncate.load())/1e9,
301  static_cast<float>(m_times.m_unlink.load())/1e9, static_cast<float>(m_times.m_rename.load())/1e9, static_cast<float>(m_times.m_chmod.load())/1e9,
302  static_cast<float>(m_slow_times.m_open.load())/1e9, static_cast<float>(m_slow_times.m_read.load())/1e9, static_cast<float>(m_slow_times.m_readv.load())/1e9,
303  static_cast<float>(m_slow_times.m_pgread.load())/1e9, static_cast<float>(m_slow_times.m_write.load())/1e9, static_cast<float>(m_slow_times.m_pgwrite.load())/1e9,
304  static_cast<float>(m_slow_times.m_dirlist.load())/1e9, static_cast<float>(m_slow_times.m_stat.load())/1e9, static_cast<float>(m_slow_times.m_truncate.load())/1e9,
305  static_cast<float>(m_slow_times.m_unlink.load())/1e9, static_cast<float>(m_slow_times.m_rename.load())/1e9, static_cast<float>(m_slow_times.m_chmod.load())/1e9
306 
307  );
308  if (len >= 1500) {
309  m_log.Log(LogMask::Error, "Aggregate", "Failed to generate g-stream statistics packet");
310  return;
311  }
312  m_log.Log(LogMask::Debug, "Aggregate", buf);
313  if (m_gstream && !m_gstream->Insert(buf, len + 1)) {
314  m_log.Log(LogMask::Error, "Aggregate", "Failed to send g-stream statistics packet");
315  return;
316  }
317 }
318 
319 StatsFileSystem::OpTimer::OpTimer(std::atomic<uint64_t> &op_count, std::atomic<uint64_t> &slow_op_count, std::atomic<uint64_t> &timing, std::atomic<uint64_t> &slow_timing, std::chrono::steady_clock::duration duration)
320  : m_op_count(op_count),
321  m_slow_op_count(slow_op_count),
322  m_timing(timing),
323  m_slow_timing(slow_timing),
324  m_start(std::chrono::steady_clock::now()),
325  m_slow_duration(duration)
326 {}
327 
328 StatsFileSystem::OpTimer::~OpTimer()
329 {
330  auto dur = std::chrono::steady_clock::now() - m_start;
331  m_op_count++;
332  m_timing += std::chrono::nanoseconds(dur).count();
333  if (dur > m_slow_duration) {
334  m_slow_op_count++;
335  m_slow_timing += std::chrono::nanoseconds(dur).count();
336  }
337 }
#define tident
int stat(const char *path, struct stat *buf)
XrdOucString Path
struct myOpts opts
bool ParseDuration(const std::string &duration, std::chrono::steady_clock::duration &result, std::string &errmsg)
std::string LogMaskToString(int mask)
@ Info
@ Warning
@ All
void Disc(XrdOucEnv &env) override
int Init(XrdSysLogger *lp, const char *cfn) override
XrdOssDF * newDir(const char *user=0) override
StatsFileSystem(XrdOss *oss, XrdSysLogger *log, const char *configName, XrdOucEnv *envP)
void EnvInfo(XrdOucEnv *env) override
void Connect(XrdOucEnv &env) override
int Stat(const char *path, struct stat *buff, int opts=0, XrdOucEnv *env=0) override
int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *env=0) override
virtual ~StatsFileSystem()
int Reloc(const char *tident, const char *path, const char *cgName, const char *anchor=0) override
int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *env=0) override
int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, int opts=0) override
XrdOssDF * newFile(const char *user=0) override
friend class StatsDirectory
bool Config(const char *configfn)
int StatPF(const char *path, struct stat *buff, int opts) override
int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen) override
int Stats(char *buff, int blen) override
int Remdir(const char *path, int Opts=0, XrdOucEnv *env=0) override
friend class StatsFile
int Unlink(const char *path, int Opts=0, XrdOucEnv *env=0) override
int Mkdir(const char *path, mode_t mode, int mkpath=0, XrdOucEnv *env=0) override
int FSctl(int cmd, int alen, const char *args, char **resp=0) override
uint64_t Features() override
int Chmod(const char *path, mode_t mode, XrdOucEnv *env=0) override
int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0) override
int Truncate(const char *path, unsigned long long fsize, XrdOucEnv *env=0) override
int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *env=0) override
int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0) override
int Lfn2Pfn(const char *Path, char *buff, int blen) override
virtual int Mkdir(const char *path, mode_t mode, int mkpath=0, XrdOucEnv *envP=0)=0
virtual int FSctl(int cmd, int alen, const char *args, char **resp=0)
Definition: XrdOss.cc:66
virtual int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen)
Definition: XrdOss.cc:97
virtual int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *envP=0)
Definition: XrdOss.cc:127
virtual int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, int opts=0)=0
virtual int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *envP=0)
Definition: XrdOss.cc:137
virtual XrdOssDF * newDir(const char *tident)=0
virtual void Connect(XrdOucEnv &env)
Definition: XrdOss.cc:42
virtual int Reloc(const char *tident, const char *path, const char *cgName, const char *anchor=0)
Definition: XrdOss.cc:76
virtual int Chmod(const char *path, mode_t mode, XrdOucEnv *envP=0)=0
virtual uint64_t Features()
Definition: XrdOss.cc:60
virtual int StatPF(const char *path, struct stat *buff, int opts)
Definition: XrdOss.cc:107
virtual void Disc(XrdOucEnv &env)
Definition: XrdOss.cc:48
virtual int Lfn2Pfn(const char *Path, char *buff, int blen)
Definition: XrdOss.hh:873
virtual int Remdir(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117
virtual int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *envP=0)
Definition: XrdOss.cc:87
virtual void EnvInfo(XrdOucEnv *envP)
Definition: XrdOss.cc:54
virtual int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0)=0
virtual XrdOssDF * newFile(const char *tident)=0
virtual int Truncate(const char *path, unsigned long long fsize, XrdOucEnv *envP=0)=0
virtual int Stats(char *buff, int blen)
Definition: XrdOss.hh:707
virtual int Stat(const char *path, struct stat *buff, int opts=0, XrdOucEnv *envP=0)=0
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
void * GetPtr(const char *varname)
Definition: XrdOucEnv.cc:263
char * GetToken(char **rest=0, int lowcase=0)
int Gather(const char *cfname, Level lvl, const char *parms=0)
@ trim_lines
Prefix trimmed lines.
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
void setMsgMask(int mask)
Definition: XrdSysError.hh:154
int getMsgMask()
Definition: XrdSysError.hh:156
void Log(int mask, const char *esfx, const char *text1, const char *text2=0, const char *text3=0)
Definition: XrdSysError.hh:133
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)
bool Insert(const char *data, int dlen)
int Opts
Definition: XrdMpxStats.cc:58
XrdOucEnv * envP
Definition: XrdPss.cc:108