XRootD
XrdPfcConfiguration.cc
Go to the documentation of this file.
1 #include "XrdPfc.hh"
2 #include "XrdPfcTrace.hh"
3 #include "XrdPfcInfo.hh"
4 
6 #include "XrdPfcPurgePin.hh"
7 
8 #include "XrdOss/XrdOss.hh"
9 
10 #include "XrdOuc/XrdOucEnv.hh"
11 #include "XrdOuc/XrdOucUtils.hh"
12 #include "XrdOuc/XrdOucStream.hh"
14 #include "XrdOuc/XrdOuca2x.hh"
15 
16 #include "XrdOfs/XrdOfsConfigPI.hh"
17 #include "XrdVersion.hh"
18 
19 #include <fcntl.h>
20 
21 using namespace XrdPfc;
22 
24 
26  m_hdfsmode(false),
27  m_allow_xrdpfc_command(false),
28  m_data_space("public"),
29  m_meta_space("public"),
30  m_diskTotalSpace(-1),
31  m_diskUsageLWM(-1),
32  m_diskUsageHWM(-1),
33  m_fileUsageBaseline(-1),
34  m_fileUsageNominal(-1),
35  m_fileUsageMax(-1),
36  m_purgeInterval(300),
37  m_purgeColdFilesAge(-1),
38  m_purgeAgeBasedPeriod(10),
39  m_accHistorySize(20),
40  m_dirStatsMaxDepth(-1),
41  m_dirStatsStoreDepth(0),
42  m_bufferSize(128*1024),
43  m_RamAbsAvailable(0),
44  m_RamKeepStdBlocks(0),
45  m_wqueue_blocks(16),
46  m_wqueue_threads(4),
47  m_prefetch_max_blocks(10),
48  m_hdfsbsize(128*1024*1024),
49  m_flushCnt(2000),
50  m_cs_UVKeep(-1),
51  m_cs_Chk(CSChk_Net),
52  m_cs_ChkTLS(false),
53  m_onlyIfCachedMinSize(1024*1024),
54  m_onlyIfCachedMinFrac(1.0)
55 {}
56 
57 
58 bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name)
59 {
60  char errStr[1024];
61  snprintf(errStr, 1024, "ConfigParameters() Error parsing parameter %s", name);
62 
63  if (::isalpha(*(str.rbegin())))
64  {
65  if (XrdOuca2x::a2sz(m_log, errStr, str.c_str(), &store, 0, totalSpace))
66  {
67  return false;
68  }
69  }
70  else
71  {
72  char *eP;
73  errno = 0;
74  double frac = strtod(str.c_str(), &eP);
75  if (errno || eP == str.c_str())
76  {
77  m_log.Emsg(errStr, str.c_str());
78  return false;
79  }
80 
81  store = static_cast<long long>(totalSpace * frac + 0.5);
82  }
83 
84  if (store < 0 || store > totalSpace)
85  {
86  snprintf(errStr, 1024, "ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)",
87  name, totalSpace, store, str.c_str());
88  m_log.Emsg(errStr, "");
89  return false;
90  }
91 
92  return true;
93 }
94 
95 /* Function: xcschk
96 
97  Purpose: To parse the directive: cschk <parms>
98 
99  parms: [[no]net] [[no]tls] [[no]cache] [uvkeep <arg>]
100 
101  all Checksum check on cache & net transfers.
102  cache Checksum check on cache only, 'no' turns it off.
103  net Checksum check on net transfers 'no' turns it off.
104  tls use TLS if server doesn't support checksums 'no' turns it off.
105  uvkeep Maximum amount of time a cached file make be kept if it
106  contains unverified checksums as n[d|h|m|s], where 'n'
107  is a non-negative integer. A value of 0 prohibits disk
108  caching unless the checksum can be verified. You can
109  also specify "lru" which means the standard purge policy
110  is to be used.
111 
112  Output: true upon success or false upon failure.
113  */
114 bool Cache::xcschk(XrdOucStream &Config)
115 {
116  const char *val, *val2;
117  struct cschkopts {const char *opname; int opval;} csopts[] =
118  {
119  {"off", CSChk_None},
120  {"cache", CSChk_Cache},
121  {"net", CSChk_Net},
122  {"tls", CSChk_TLS}
123  };
124  int i, numopts = sizeof(csopts)/sizeof(struct cschkopts);
125  bool isNo;
126 
127  if (! (val = Config.GetWord()))
128  {m_log.Emsg("Config", "cschk parameter not specified"); return false; }
129 
130  while(val)
131  {
132  if ((isNo = strncmp(val, "no", 2) == 0))
133  val2 = val + 2;
134  else
135  val2 = val;
136  for (i = 0; i < numopts; i++)
137  {
138  if (!strcmp(val2, csopts[i].opname))
139  {
140  if (isNo)
141  m_configuration.m_cs_Chk &= ~csopts[i].opval;
142  else if (csopts[i].opval)
143  m_configuration.m_cs_Chk |= csopts[i].opval;
144  else
145  m_configuration.m_cs_Chk = csopts[i].opval;
146  break;
147  }
148  }
149  if (i >= numopts)
150  {
151  if (strcmp(val, "uvkeep"))
152  {
153  m_log.Emsg("Config", "invalid cschk option -", val);
154  return false;
155  }
156  if (!(val = Config.GetWord()))
157  {
158  m_log.Emsg("Config", "cschk uvkeep value not specified");
159  return false;
160  }
161  if (!strcmp(val, "lru"))
162  m_configuration.m_cs_UVKeep = -1;
163  else
164  {
165  int uvkeep;
166  if (XrdOuca2x::a2tm(m_log, "uvkeep time", val, &uvkeep, 0))
167  return false;
168  m_configuration.m_cs_UVKeep = uvkeep;
169  }
170  }
171  val = Config.GetWord();
172  }
173  // Decompose into separate TLS state, it is only passed on to psx
174  m_configuration.m_cs_ChkTLS = m_configuration.m_cs_Chk & CSChk_TLS;
175  m_configuration.m_cs_Chk &= ~CSChk_TLS;
176 
177  m_env->Put("psx.CSNet", m_configuration.is_cschk_net() ? (m_configuration.m_cs_ChkTLS ? "2" : "1") : "0");
178 
179  return true;
180 }
181 
182 
183 /* Function: xdlib
184 
185  Purpose: To parse the directive: decisionlib <path> [<parms>]
186 
187  <path> the path of the decision library to be used.
188  <parms> optional parameters to be passed.
189 
190 
191  Output: true upon success or false upon failure.
192  */
193 bool Cache::xdlib(XrdOucStream &Config)
194 {
195  const char* val;
196 
197  std::string libp;
198  if (! (val = Config.GetWord()) || ! val[0])
199  {
200  TRACE(Info," Cache::Config() decisionlib not specified; always caching files");
201  return true;
202  }
203  else
204  {
205  libp = val;
206  }
207 
208  char params[4096];
209  if (val[0])
210  Config.GetRest(params, 4096);
211  else
212  params[0] = 0;
213 
214  XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib",
215  libp.c_str());
216 
217  Decision *(*ep)(XrdSysError&);
218  ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetDecision");
219  if (! ep) {myLib->Unload(true); return false; }
220 
221  Decision * d = ep(m_log);
222  if (! d)
223  {
224  TRACE(Error, "Config() decisionlib was not able to create a decision object");
225  return false;
226  }
227  if (params[0])
228  d->ConfigDecision(params);
229 
230  m_decisionpoints.push_back(d);
231  return true;
232 }
233 
234 /* Function: xplib
235 
236  Purpose: To parse the directive: purgelib <path> [<parms>]
237 
238  <path> the path of the decision library to be used.
239  <parms> optional parameters to be passed.
240 
241 
242  Output: true upon success or false upon failure.
243  */
244 bool Cache::xplib(XrdOucStream &Config)
245 {
246  const char* val;
247 
248  std::string libp;
249  if (! (val = Config.GetWord()) || ! val[0])
250  {
251  TRACE(Info," Cache::Config() purgelib not specified; will use LRU for purging files");
252  return true;
253  }
254  else
255  {
256  libp = val;
257  }
258 
259  char params[4096];
260  if (val[0])
261  Config.GetRest(params, 4096);
262  else
263  params[0] = 0;
264 
265  XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "purgelib",
266  libp.c_str());
267 
268  PurgePin *(*ep)(XrdSysError&);
269  ep = (PurgePin *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetPurgePin");
270  if (! ep) {myLib->Unload(true); return false; }
271 
272  PurgePin * dp = ep(m_log);
273  if (! dp)
274  {
275  TRACE(Error, "Config() purgelib was not able to create a Purge Plugin object?");
276  return false;
277  }
278  m_purge_pin = dp;
279 
280  if (params[0])
281  m_purge_pin->ConfigPurgePin(params);
282 
283 
284  return true;
285 }
286 
287 /* Function: xtrace
288 
289  Purpose: To parse the directive: trace <level>
290  Output: true upon success or false upon failure.
291  */
292 bool Cache::xtrace(XrdOucStream &Config)
293 {
294  char *val;
295  static struct traceopts {const char *opname; int opval; } tropts[] =
296  {
297  {"none", 0},
298  {"error", 1},
299  {"warning", 2},
300  {"info", 3},
301  {"debug", 4},
302  {"dump", 5},
303  {"dumpxl", 6}
304  };
305  int numopts = sizeof(tropts)/sizeof(struct traceopts);
306 
307  if (! (val = Config.GetWord()))
308  {m_log.Emsg("Config", "trace option not specified"); return 1; }
309 
310  for (int i = 0; i < numopts; i++)
311  {
312  if (! strcmp(val, tropts[i].opname))
313  {
314  m_trace->What = tropts[i].opval;
315  return true;
316  }
317  }
318  m_log.Emsg("Config", "invalid trace option -", val);
319  return false;
320 }
321 
322 
323 //______________________________________________________________________________
324 /* Function: Config
325 
326  Purpose: To parse configuration file and configure Cache instance.
327  Output: true upon success or false upon failure.
328  */
329 bool Cache::Config(const char *config_filename, const char *parameters)
330 {
331  // Indicate whether or not we are a client instance
332  const char *theINS = getenv("XRDINSTANCE");
333  m_isClient = (theINS != 0 && strncmp("*client ", theINS, 8) == 0);
334 
335  // Tell everyone else we are a caching proxy
336  XrdOucEnv::Export("XRDPFC", 1);
337 
338  XrdOucEnv myEnv;
339  XrdOucStream Config(&m_log, theINS, &myEnv, "=====> ");
340 
341  if (! config_filename || ! *config_filename)
342  {
343  TRACE(Error, "Config() configuration file not specified.");
344  return false;
345  }
346 
347  int fd;
348  if ( (fd = open(config_filename, O_RDONLY, 0)) < 0)
349  {
350  TRACE( Error, "Config() can't open configuration file " << config_filename);
351  return false;
352  }
353 
354  Config.Attach(fd);
355  static const char *cvec[] = { "*** pfc plugin config:", 0 };
356  Config.Capture(cvec);
357 
358  // Obtain OFS configurator for OSS plugin.
359  XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log,
360  &XrdVERSIONINFOVAR(XrdOucGetCache));
361  if (! ofsCfg) return false;
362 
363  TmpConfiguration tmpc;
364 
365  // Adjust default parameters for client/serverless caching
366  if (m_isClient)
367  {
368  m_configuration.m_bufferSize = 128 * 1024; // same as normal.
369  m_configuration.m_wqueue_blocks = 8;
370  m_configuration.m_wqueue_threads = 1;
371  }
372 
373  // If network checksum processing is the default, indicate so.
374  if (m_configuration.is_cschk_net()) m_env->Put("psx.CSNet", m_configuration.m_cs_ChkTLS ? "2" : "1");
375 
376  // Actual parsing of the config file.
377  bool retval = true, aOK = true;
378  char *var;
379  while ((var = Config.GetMyFirstWord()))
380  {
381  if (! strcmp(var,"pfc.osslib"))
382  {
383  retval = ofsCfg->Parse(XrdOfsConfigPI::theOssLib);
384  }
385  else if (! strcmp(var,"pfc.cschk"))
386  {
387  retval = xcschk(Config);
388  }
389  else if (! strcmp(var,"pfc.decisionlib"))
390  {
391  retval = xdlib(Config);
392  }
393  else if (! strcmp(var,"pfc.purgelib"))
394  {
395  retval = xplib(Config);
396  }
397  else if (! strcmp(var,"pfc.trace"))
398  {
399  retval = xtrace(Config);
400  }
401  else if (! strcmp(var,"pfc.allow_xrdpfc_command"))
402  {
403  m_configuration.m_allow_xrdpfc_command = true;
404  }
405  else if (! strncmp(var,"pfc.", 4))
406  {
407  retval = ConfigParameters(std::string(var+4), Config, tmpc);
408  }
409 
410  if ( ! retval)
411  {
412  TRACE(Error, "Config() error in parsing");
413  aOK = false;
414  }
415  }
416 
417  Config.Close();
418 
419  // Load OSS plugin.
420  myEnv.Put("oss.runmode", "pfc");
421  if (m_configuration.is_cschk_cache())
422  {
423  char csi_conf[128];
424  if (snprintf(csi_conf, 128, "space=%s nofill", m_configuration.m_meta_space.c_str()) < 128)
425  {
426  ofsCfg->Push(XrdOfsConfigPI::theOssLib, "libXrdOssCsi.so", csi_conf);
427  } else {
428  TRACE(Error, "Config() buffer too small for libXrdOssCsi params.");
429  return false;
430  }
431  }
432  if (ofsCfg->Load(XrdOfsConfigPI::theOssLib, &myEnv))
433  {
434  ofsCfg->Plugin(m_oss);
435  }
436  else
437  {
438  TRACE(Error, "Config() Unable to create an OSS object");
439  return false;
440  }
441 
442  // sets default value for disk usage
443  XrdOssVSInfo sP;
444  {
445  if (m_configuration.m_meta_space != m_configuration.m_data_space &&
446  m_oss->StatVS(&sP, m_configuration.m_meta_space.c_str(), 1) < 0)
447  {
448  m_log.Emsg("ConfigParameters()", "error obtaining stat info for meta space ", m_configuration.m_meta_space.c_str());
449  return false;
450  }
451  if (m_configuration.m_meta_space != m_configuration.m_data_space && sP.Total < 10ll << 20)
452  {
453  m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ",
454  m_configuration.m_meta_space.c_str());
455  return false;
456  }
457  if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0)
458  {
459  m_log.Emsg("ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str());
460  return false;
461  }
462  if (sP.Total < 10ll << 20)
463  {
464  m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ",
465  m_configuration.m_data_space.c_str());
466  return false;
467  }
468 
469  m_configuration.m_diskTotalSpace = sP.Total;
470 
471  if (cfg2bytes(tmpc.m_diskUsageLWM, m_configuration.m_diskUsageLWM, sP.Total, "lowWatermark") &&
472  cfg2bytes(tmpc.m_diskUsageHWM, m_configuration.m_diskUsageHWM, sP.Total, "highWatermark"))
473  {
474  if (m_configuration.m_diskUsageLWM >= m_configuration.m_diskUsageHWM) {
475  m_log.Emsg("ConfigParameters()", "pfc.diskusage should have lowWatermark < highWatermark.");
476  aOK = false;
477  }
478  }
479  else aOK = false;
480 
481  if ( ! tmpc.m_fileUsageMax.empty())
482  {
483  if (cfg2bytes(tmpc.m_fileUsageBaseline, m_configuration.m_fileUsageBaseline, sP.Total, "files baseline") &&
484  cfg2bytes(tmpc.m_fileUsageNominal, m_configuration.m_fileUsageNominal, sP.Total, "files nominal") &&
485  cfg2bytes(tmpc.m_fileUsageMax, m_configuration.m_fileUsageMax, sP.Total, "files max"))
486  {
487  if (m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageNominal ||
488  m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageMax ||
489  m_configuration.m_fileUsageNominal >= m_configuration.m_fileUsageMax)
490  {
491  m_log.Emsg("ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max.");
492  aOK = false;
493  }
494 
495 
496  if (aOK && m_configuration.m_fileUsageMax >= m_configuration.m_diskUsageLWM)
497  {
498  m_log.Emsg("ConfigParameters()", "pfc.diskusage files values must be below lowWatermark");
499  aOK = false;
500  }
501  }
502  else aOK = false;
503  }
504  }
505  // sets flush frequency
506  if ( ! tmpc.m_flushRaw.empty())
507  {
508  if (::isalpha(*(tmpc.m_flushRaw.rbegin())))
509  {
510  if (XrdOuca2x::a2sz(m_log, "Error getting number of bytes written before flush", tmpc.m_flushRaw.c_str(),
511  &m_configuration.m_flushCnt,
512  100 * m_configuration.m_bufferSize , 100000 * m_configuration.m_bufferSize))
513  {
514  return false;
515  }
516  m_configuration.m_flushCnt /= m_configuration.m_bufferSize;
517  }
518  else
519  {
520  if (XrdOuca2x::a2ll(m_log, "Error getting number of blocks written before flush", tmpc.m_flushRaw.c_str(),
521  &m_configuration.m_flushCnt, 100, 100000))
522  {
523  return false;
524  }
525  }
526  }
527 
528  // get number of available RAM blocks after process configuration
529  if (m_configuration.m_RamAbsAvailable == 0)
530  {
531  m_configuration.m_RamAbsAvailable = m_isClient ? 256ll * 1024 * 1024 : 1024ll * 1024 * 1024;
532  char buff[1024];
533  snprintf(buff, sizeof(buff), "RAM usage pfc.ram is not specified. Default value %s is used.", m_isClient ? "256m" : "1g");
534  m_log.Say("Config info: ", buff);
535  }
536  // Setup number of standard-size blocks not released back to the system to 5% of total RAM.
537  m_configuration.m_RamKeepStdBlocks = (m_configuration.m_RamAbsAvailable / m_configuration.m_bufferSize + 1) * 5 / 100;
538 
539 
540  // Set tracing to debug if this is set in environment
541  char* cenv = getenv("XRDDEBUG");
542  if (cenv && ! strcmp(cenv,"1") && m_trace->What < 4) m_trace->What = 4;
543 
544  if (aOK)
545  {
546  int loff = 0;
547 // 000 001 010
548  const char *csc[] = {"off", "cache nonet", "nocache net notls",
549 // 011
550  "cache net notls",
551 // 100 101 110
552  "off", "cache nonet", "nocache net tls",
553 // 111
554  "cache net tls"};
555  char buff[8192], uvk[32];
556  if (m_configuration.m_cs_UVKeep < 0)
557  strcpy(uvk, "lru");
558  else
559  sprintf(uvk, "%lld", (long long) m_configuration.m_cs_UVKeep);
560  float rg = (m_configuration.m_RamAbsAvailable) / float(1024*1024*1024);
561  loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n"
562  " pfc.cschk %s uvkeep %s\n"
563  " pfc.blocksize %lld\n"
564  " pfc.prefetch %d\n"
565  " pfc.ram %.fg\n"
566  " pfc.writequeue %d %d\n"
567  " # Total available disk: %lld\n"
568  " pfc.diskusage %lld %lld files %lld %lld %lld purgeinterval %d purgecoldfiles %d\n"
569  " pfc.spaces %s %s\n"
570  " pfc.trace %d\n"
571  " pfc.flush %lld\n"
572  " pfc.acchistorysize %d\n"
573  " pfc.onlyIfCachedMinBytes %lld\n"
574  " pfc.onlyIfCachedMinFrac %.2f\n",
575  config_filename,
576  csc[int(m_configuration.m_cs_Chk)], uvk,
577  m_configuration.m_bufferSize,
578  m_configuration.m_prefetch_max_blocks,
579  rg,
580  m_configuration.m_wqueue_blocks, m_configuration.m_wqueue_threads,
581  sP.Total,
582  m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM,
583  m_configuration.m_fileUsageBaseline, m_configuration.m_fileUsageNominal, m_configuration.m_fileUsageMax,
584  m_configuration.m_purgeInterval, m_configuration.m_purgeColdFilesAge,
585  m_configuration.m_data_space.c_str(),
586  m_configuration.m_meta_space.c_str(),
587  m_trace->What,
588  m_configuration.m_flushCnt,
589  m_configuration.m_accHistorySize,
590  m_configuration.m_onlyIfCachedMinSize,
591  m_configuration.m_onlyIfCachedMinFrac);
592 
593  if (m_configuration.is_dir_stat_reporting_on())
594  {
595  loff += snprintf(buff + loff, sizeof(buff) - loff,
596  " pfc.dirstats maxdepth %d ((internal: store_depth %d, size_of_dirlist %d, size_of_globlist %d))\n",
597  m_configuration.m_dirStatsMaxDepth, m_configuration.m_dirStatsStoreDepth,
598  (int) m_configuration.m_dirStatsDirs.size(), (int) m_configuration.m_dirStatsDirGlobs.size());
599  loff += snprintf(buff + loff, sizeof(buff) - loff, " dirlist:\n");
600  for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirs.begin(); i != m_configuration.m_dirStatsDirs.end(); ++i)
601  loff += snprintf(buff + loff, sizeof(buff) - loff, " %s\n", i->c_str());
602  loff += snprintf(buff + loff, sizeof(buff) - loff, " globlist:\n");
603  for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirGlobs.begin(); i != m_configuration.m_dirStatsDirGlobs.end(); ++i)
604  loff += snprintf(buff + loff, sizeof(buff) - loff, " %s/*\n", i->c_str());
605  }
606 
607  if (m_configuration.m_hdfsmode)
608  {
609  loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize);
610  }
611 
612  if (m_configuration.m_username.empty())
613  {
614  char unameBuff[256];
615  XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff));
616  m_configuration.m_username = unameBuff;
617  }
618  else
619  {
620  loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.user %s\n", m_configuration.m_username.c_str());
621  }
622 
623  m_log.Say(buff);
624 
625  m_env->Put("XRDPFC.SEGSIZE", std::to_string(m_configuration.m_bufferSize).c_str());
626  }
627 
628  // Derived settings
629  m_prefetch_enabled = m_configuration.m_prefetch_max_blocks > 0;
630  Info::s_maxNumAccess = m_configuration.m_accHistorySize;
631 
632  m_gstream = (XrdXrootdGStream*) m_env->GetPtr("pfc.gStream*");
633 
634  m_log.Say(" pfc g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive\n");
635 
636  // Create the ResourceMonitor and get it ready for starting the main thread function.
637  if (aOK)
638  {
639  m_res_mon = new ResourceMonitor(*m_oss);
640  m_res_mon->init_before_main();
641  }
642 
643  m_log.Say("=====> Proxy file cache configuration parsing ", aOK ? "completed" : "failed");
644 
645  if (ofsCfg) delete ofsCfg;
646 
647  // XXXX-CKSUM Testing. To be removed after OssPgi is also merged and valildated.
648  // Building of xrdpfc_print fails when this is enabled.
649 #ifdef XRDPFC_CKSUM_TEST
650  {
651  int xxx = m_configuration.m_cs_Chk;
652 
653  for (m_configuration.m_cs_Chk = CSChk_None; m_configuration.m_cs_Chk <= CSChk_Both; ++m_configuration.m_cs_Chk)
654  {
655  Info::TestCksumStuff();
656  }
657 
658  m_configuration.m_cs_Chk = xxx;
659  }
660 #endif
661 
662  return aOK;
663 }
664 
665 //------------------------------------------------------------------------------
666 
667 bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfiguration &tmpc)
668 {
669  struct ConfWordGetter
670  {
671  XrdOucStream &m_config;
672  char *m_last_word;
673 
674  ConfWordGetter(XrdOucStream& c) : m_config(c), m_last_word((char*)1) {}
675 
676  const char* GetWord() { if (HasLast()) m_last_word = m_config.GetWord(); return HasLast() ? m_last_word : ""; }
677  bool HasLast() { return (m_last_word != 0); }
678  };
679 
680  ConfWordGetter cwg(config);
681 
682  XrdSysError err(0, "");
683  if ( part == "user" )
684  {
685  m_configuration.m_username = cwg.GetWord();
686  if ( ! cwg.HasLast())
687  {
688  m_log.Emsg("Config", "Error: pfc.user requires a parameter.");
689  return false;
690  }
691  }
692  else if ( part == "diskusage" )
693  {
694  tmpc.m_diskUsageLWM = cwg.GetWord();
695  tmpc.m_diskUsageHWM = cwg.GetWord();
696 
697  if (tmpc.m_diskUsageHWM.empty())
698  {
699  m_log.Emsg("Config", "Error: pfc.diskusage parameter requires at least two arguments.");
700  return false;
701  }
702 
703  const char *p = 0;
704  while ((p = cwg.GetWord()) && cwg.HasLast())
705  {
706  if (strcmp(p, "files") == 0)
707  {
708  tmpc.m_fileUsageBaseline = cwg.GetWord();
709  tmpc.m_fileUsageNominal = cwg.GetWord();
710  tmpc.m_fileUsageMax = cwg.GetWord();
711 
712  if ( ! cwg.HasLast())
713  {
714  m_log.Emsg("Config", "Error: pfc.diskusage files directive requires three arguments.");
715  return false;
716  }
717  }
718  else if (strcmp(p, "sleep") == 0 || strcmp(p, "purgeinterval") == 0)
719  {
720  if (strcmp(p, "sleep") == 0) m_log.Emsg("Config", "warning sleep directive is deprecated in pfc.diskusage. Please use purgeinterval instead.");
721 
722  if (XrdOuca2x::a2tm(m_log, "Error getting purgeinterval", cwg.GetWord(), &m_configuration.m_purgeInterval, 60, 3600))
723  {
724  return false;
725  }
726  }
727  else if (strcmp(p, "purgecoldfiles") == 0)
728  {
729  if (XrdOuca2x::a2tm(m_log, "Error getting purgecoldfiles age", cwg.GetWord(), &m_configuration.m_purgeColdFilesAge, 3600, 3600*24*360))
730  {
731  return false;
732  }
733  if (XrdOuca2x::a2i(m_log, "Error getting purgecoldfiles period", cwg.GetWord(), &m_configuration.m_purgeAgeBasedPeriod, 1, 1000))
734  {
735  return false;
736  }
737  }
738  else
739  {
740  m_log.Emsg("Config", "Error: diskusage stanza contains unknown directive", p);
741  }
742  }
743  }
744  else if ( part == "acchistorysize" )
745  {
746  if ( XrdOuca2x::a2i(m_log, "Error getting access-history-size", cwg.GetWord(), &m_configuration.m_accHistorySize, 20, 200))
747  {
748  return false;
749  }
750  }
751  else if ( part == "dirstats" )
752  {
753  const char *p = 0;
754  while ((p = cwg.GetWord()) && cwg.HasLast())
755  {
756  if (strcmp(p, "maxdepth") == 0)
757  {
758  if (XrdOuca2x::a2i(m_log, "Error getting maxdepth value", cwg.GetWord(), &m_configuration.m_dirStatsMaxDepth, 0, 16))
759  {
760  return false;
761  }
762  m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, m_configuration.m_dirStatsMaxDepth);
763  }
764  else if (strcmp(p, "dir") == 0)
765  {
766  p = cwg.GetWord();
767  if (p && p[0] == '/')
768  {
769  // XXX -- should we just store them as sets of PathTokenizer objects, not strings?
770 
771  char d[1024]; d[0] = 0;
772  int depth = 0;
773  { // Compress multiple slashes and "measure" depth
774  const char *pp = p;
775  char *pd = d;
776  *(pd++) = *(pp++);
777  while (*pp != 0)
778  {
779  if (*(pd - 1) == '/')
780  {
781  if (*pp == '/')
782  {
783  ++pp; continue;
784  }
785  ++depth;
786  }
787  *(pd++) = *(pp++);
788  }
789  *(pd--) = 0;
790  // remove trailing but but not leading /
791  if (*pd == '/' && pd != d) *pd = 0;
792  }
793  int ld = strlen(d);
794  if (ld >= 2 && d[ld-1] == '*' && d[ld-2] == '/')
795  {
796  d[ld-2] = 0;
797  ld -= 2;
798  m_configuration.m_dirStatsDirGlobs.insert(d);
799  printf("Glob %s -> %s -- depth = %d\n", p, d, depth);
800  }
801  else
802  {
803  m_configuration.m_dirStatsDirs.insert(d);
804  printf("Dir %s -> %s -- depth = %d\n", p, d, depth);
805  }
806 
807  m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, depth);
808  }
809  else
810  {
811  m_log.Emsg("Config", "Error: dirstats dir parameter requires a directory argument starting with a '/'.");
812  return false;
813  }
814  }
815  else
816  {
817  m_log.Emsg("Config", "Error: dirstats stanza contains unknown directive '", p, "'");
818  return false;
819  }
820  }
821  }
822  else if ( part == "blocksize" )
823  {
824  long long minBSize = 4 * 1024;
825  long long maxBSize = 512 * 1024 * 1024;
826  if (XrdOuca2x::a2sz(m_log, "Error reading block-size", cwg.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize))
827  {
828  return false;
829  }
830  if (m_configuration.m_bufferSize & 0xFFF)
831  {
832  m_configuration.m_bufferSize &= ~0x0FFF;
833  m_configuration.m_bufferSize += 0x1000;
834  m_log.Emsg("Config", "pfc.blocksize must be a multiple of 4 kB. Rounded up.");
835  }
836  }
837  else if ( part == "prefetch" || part == "nramprefetch" )
838  {
839  if (part == "nramprefetch")
840  {
841  m_log.Emsg("Config", "pfc.nramprefetch is deprecated, please use pfc.prefetch instead. Replacing the directive internally.");
842  }
843 
844  if (XrdOuca2x::a2i(m_log, "Error setting prefetch block count", cwg.GetWord(), &m_configuration.m_prefetch_max_blocks, 0, 128))
845  {
846  return false;
847  }
848 
849  }
850  else if ( part == "nramread" )
851  {
852  m_log.Emsg("Config", "pfc.nramread is deprecated, please use pfc.ram instead. Ignoring this directive.");
853  cwg.GetWord(); // Ignoring argument.
854  }
855  else if ( part == "ram" )
856  {
857  long long minRAM = m_isClient ? 256 * 1024 * 1024 : 1024 * 1024 * 1024;
858  long long maxRAM = 256 * minRAM;
859  if ( XrdOuca2x::a2sz(m_log, "get RAM available", cwg.GetWord(), &m_configuration.m_RamAbsAvailable, minRAM, maxRAM))
860  {
861  return false;
862  }
863  }
864  else if ( part == "writequeue")
865  {
866  if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-blocks", cwg.GetWord(), &m_configuration.m_wqueue_blocks, 1, 1024))
867  {
868  return false;
869  }
870  if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-threads", cwg.GetWord(), &m_configuration.m_wqueue_threads, 1, 64))
871  {
872  return false;
873  }
874  }
875  else if ( part == "spaces" )
876  {
877  m_configuration.m_data_space = cwg.GetWord();
878  m_configuration.m_meta_space = cwg.GetWord();
879  if ( ! cwg.HasLast())
880  {
881  m_log.Emsg("Config", "spacenames requires two parameters: <data-space> <metadata-space>.");
882  return false;
883  }
884  }
885  else if ( part == "hdfsmode" )
886  {
887  m_log.Emsg("Config", "pfc.hdfsmode is currently unsupported.");
888  return false;
889 
890  m_configuration.m_hdfsmode = true;
891 
892  const char* params = cwg.GetWord();
893  if (params)
894  {
895  if (! strncmp("hdfsbsize", params, 9))
896  {
897  long long minBlSize = 32 * 1024;
898  long long maxBlSize = 128 * 1024 * 1024;
899  if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", cwg.GetWord(), &m_configuration.m_hdfsbsize, minBlSize, maxBlSize))
900  {
901  return false;
902  }
903  }
904  else
905  {
906  m_log.Emsg("Config", "Error setting the fragment size parameter name");
907  return false;
908  }
909  }
910  }
911  else if ( part == "flush" )
912  {
913  tmpc.m_flushRaw = cwg.GetWord();
914  if ( ! cwg.HasLast())
915  {
916  m_log.Emsg("Config", "Error: pfc.flush requires a parameter.");
917  return false;
918  }
919  }
920  else if ( part == "onlyifcached" )
921  {
922  const char *p = 0;
923  while ((p = cwg.GetWord()) && cwg.HasLast())
924  {
925  if (strcmp(p, "minsize") == 0)
926  {
927  std::string minBytes = cwg.GetWord();
928  long long minBytesTop = 1024 * 1024 * 1024;
929  if (::isalpha(*(minBytes.rbegin())))
930  {
931  if (XrdOuca2x::a2sz(m_log, "Error in parsing minsize value for onlyifcached parameter", minBytes.c_str(), &m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
932  {
933  return false;
934  }
935  }
936  else
937  {
938  if (XrdOuca2x::a2ll(m_log, "Error in parsing numeric minsize value for onlyifcached parameter", minBytes.c_str(),&m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
939  {
940  return false;
941  }
942  }
943  }
944  if (strcmp(p, "minfrac") == 0)
945  {
946  std::string minFrac = cwg.GetWord();
947  char *eP;
948  errno = 0;
949  double frac = strtod(minFrac.c_str(), &eP);
950  if (errno || eP == minFrac.c_str())
951  {
952  m_log.Emsg("Config", "Error setting fraction for only-if-cached directive");
953  return false;
954  }
955  m_configuration.m_onlyIfCachedMinFrac = frac;
956  }
957  else
958  {
959  m_log.Emsg("Config", "Error: onlyifcached stanza contains unknown directive", p);
960  }
961  }
962  }
963  else
964  {
965  m_log.Emsg("ConfigParameters() unmatched pfc parameter", part.c_str());
966  return false;
967  }
968 
969  return true;
970 }
XrdVERSIONINFO(XrdOucGetCache, XrdPfc)
XrdOucCache * XrdOucGetCache(XrdSysLogger *logger, const char *config_filename, const char *parameters, XrdOucEnv *env)
Definition: XrdPfc.cc:74
int open(const char *path, int oflag,...)
int isNo(int dflt, const char *Msg1, const char *Msg2, const char *Msg3)
if(Avsz)
#define TRACE(act, x)
Definition: XrdTrace.hh:63
bool Parse(TheLib what)
bool Plugin(XrdAccAuthorize *&piP)
Get Authorization plugin.
static XrdOfsConfigPI * New(const char *cfn, XrdOucStream *cfgP, XrdSysError *errP, XrdVersionInfo *verP=0, XrdSfsFileSystem *sfsP=0)
bool Load(int what, XrdOucEnv *envP=0)
bool Push(TheLib what, const char *plugP, const char *parmP=0)
@ theOssLib
Oss plugin.
long long Total
Definition: XrdOssVS.hh:90
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117
static int Export(const char *Var, const char *Val)
Definition: XrdOucEnv.cc:170
void * GetPtr(const char *varname)
Definition: XrdOucEnv.cc:263
void Put(const char *varname, const char *value)
Definition: XrdOucEnv.hh:85
void * Resolve(const char *symbl, int mcnt=1)
void Unload(bool dodel=false)
char * GetWord(int lowcase=0)
static int UserName(uid_t uID, char *uName, int uNsz)
static int a2i(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition: XrdOuca2x.cc:45
static int a2sz(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition: XrdOuca2x.cc:257
static int a2ll(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition: XrdOuca2x.cc:70
static int a2tm(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition: XrdOuca2x.cc:288
bool Config(const char *config_filename, const char *parameters)
Parse configuration file.
Base class for selecting which files should be cached.
virtual bool ConfigDecision(const char *params)
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:311
Base class for reguesting directory space to obtain.
virtual bool ConfigPurgePin(const char *params)
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
XrdCmsConfig Config
Definition: XrdPfc.hh:41
@ CSChk_Both
Definition: XrdPfcTypes.hh:27
@ CSChk_Net
Definition: XrdPfcTypes.hh:27
@ CSChk_TLS
Definition: XrdPfcTypes.hh:28
@ CSChk_Cache
Definition: XrdPfcTypes.hh:27
@ CSChk_None
Definition: XrdPfcTypes.hh:27
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
Definition: XrdPfc.hh:114
long long m_RamAbsAvailable
available from configuration
Definition: XrdPfc.hh:108
long long m_flushCnt
nuber of unsynced blcoks on disk before flush is called
Definition: XrdPfc.hh:115
int m_accHistorySize
max number of entries in access history part of cinfo file
Definition: XrdPfc.hh:100
int m_wqueue_threads
number of threads writing blocks to disk
Definition: XrdPfc.hh:111
long long m_diskTotalSpace
total disk space on configured partition or oss space
Definition: XrdPfc.hh:91
long long m_fileUsageMax
cache purge - files usage maximum
Definition: XrdPfc.hh:96
long long m_fileUsageBaseline
cache purge - files usage baseline
Definition: XrdPfc.hh:94
int m_dirStatsStoreDepth
depth to which statistics should be collected
Definition: XrdPfc.hh:105
bool m_allow_xrdpfc_command
flag for enabling access to /xrdpfc-command/ functionality.
Definition: XrdPfc.hh:85
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition: XrdPfc.hh:93
bool is_cschk_cache() const
Definition: XrdPfc.hh:75
std::set< std::string > m_dirStatsDirGlobs
directory globs for which stat reporting was requested
Definition: XrdPfc.hh:103
int m_prefetch_max_blocks
maximum number of blocks to prefetch per file
Definition: XrdPfc.hh:112
bool m_cs_ChkTLS
Allow TLS.
Definition: XrdPfc.hh:119
long long m_fileUsageNominal
cache purge - files usage nominal
Definition: XrdPfc.hh:95
int m_cs_Chk
Checksum check.
Definition: XrdPfc.hh:118
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition: XrdPfc.hh:99
bool m_hdfsmode
flag for enabling block-level operation
Definition: XrdPfc.hh:84
int m_purgeColdFilesAge
purge files older than this age
Definition: XrdPfc.hh:98
std::string m_data_space
oss space for data files
Definition: XrdPfc.hh:88
std::set< std::string > m_dirStatsDirs
directories for which stat reporting was requested
Definition: XrdPfc.hh:102
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition: XrdPfc.hh:92
int m_RamKeepStdBlocks
number of standard-sized blocks kept after release
Definition: XrdPfc.hh:109
long long m_bufferSize
prefetch buffer size, default 1MB
Definition: XrdPfc.hh:107
std::string m_meta_space
oss space for metadata files (cinfo)
Definition: XrdPfc.hh:89
int m_wqueue_blocks
maximum number of blocks written per write-queue loop
Definition: XrdPfc.hh:110
std::string m_username
username passed to oss plugin
Definition: XrdPfc.hh:87
bool is_cschk_net() const
Definition: XrdPfc.hh:76
double m_onlyIfCachedMinFrac
minimum fraction of downloaded file, used by only-if-cached CGI option
Definition: XrdPfc.hh:122
time_t m_cs_UVKeep
unverified checksum cache keep
Definition: XrdPfc.hh:117
int m_dirStatsMaxDepth
maximum depth for statistics write out
Definition: XrdPfc.hh:104
int m_purgeInterval
sleep interval between cache purges
Definition: XrdPfc.hh:97
long long m_onlyIfCachedMinSize
minumum size of downloaded file, used by only-if-cached CGI option
Definition: XrdPfc.hh:121
bool is_dir_stat_reporting_on() const
Definition: XrdPfc.hh:70
std::string m_diskUsageLWM
Definition: XrdPfc.hh:129
std::string m_diskUsageHWM
Definition: XrdPfc.hh:130
std::string m_fileUsageBaseline
Definition: XrdPfc.hh:131
std::string m_fileUsageNominal
Definition: XrdPfc.hh:132
std::string m_flushRaw
Definition: XrdPfc.hh:134
std::string m_fileUsageMax
Definition: XrdPfc.hh:133