XRootD
XrdPfc::FPurgeState Class Reference
+ Collaboration diagram for XrdPfc::FPurgeState:

Classes

struct  FS
 

Public Types

typedef list_t::iterator list_i
 
typedef std::list< FSlist_t
 
typedef map_t::iterator map_i
 
typedef std::multimap< time_t, FSmap_t
 

Public Member Functions

 FPurgeState (long long iNBytesReq, XrdOss &oss)
 
void begin_traversal (DirState *root, const char *root_path="/")
 
void cd_down (const std::string &dir_name)
 
void cd_up ()
 
void CheckFile (const char *fname, Info &info, struct stat &fstat)
 
void end_traversal ()
 
time_t getMinTime () const
 
long long getNBytesTotal () const
 
void MoveListEntriesToMap ()
 
void setMinTime (time_t min_time)
 
void setUVKeepMinTime (time_t min_time)
 
void TraverseNamespace (XrdOssDF *iOssDF)
 

Public Attributes

std::string m_current_path
 
int m_dir_level
 
std::vector< std::string > m_dir_names_stack
 
DirStatem_dir_state
 
std::vector< long long > m_dir_usage_stack
 
list_t m_flist
 
map_t m_fmap
 
const char * m_info_ext
 
const size_t m_info_ext_len
 
const int m_max_dir_level_for_stat_collection
 
XrdOssAt m_oss_at
 
XrdSysTracem_trace
 
long long nBytesAccum
 
long long nBytesReq
 
long long nBytesTotal
 
time_t tMinTimeStamp
 
time_t tMinUVKeepTimeStamp
 

Static Public Attributes

static const char * m_traceID = "Purge"
 

Detailed Description

Definition at line 222 of file XrdPfcPurge.cc.

Member Typedef Documentation

◆ list_i

typedef list_t::iterator XrdPfc::FPurgeState::list_i

Definition at line 243 of file XrdPfcPurge.cc.

◆ list_t

typedef std::list<FS> XrdPfc::FPurgeState::list_t

Definition at line 242 of file XrdPfcPurge.cc.

◆ map_i

typedef map_t::iterator XrdPfc::FPurgeState::map_i

Definition at line 238 of file XrdPfcPurge.cc.

◆ map_t

typedef std::multimap<time_t, FS> XrdPfc::FPurgeState::map_t

Definition at line 237 of file XrdPfcPurge.cc.

Constructor & Destructor Documentation

◆ FPurgeState()

XrdPfc::FPurgeState::FPurgeState ( long long  iNBytesReq,
XrdOss oss 
)
inline

Definition at line 334 of file XrdPfcPurge.cc.

334  :
336  // m_oss(oss),
337  m_oss_at(oss),
338  m_dir_state(0), m_dir_level(0),
339  m_max_dir_level_for_stat_collection(Cache::Conf().m_dirStatsStoreDepth),
343  {
344  m_current_path.reserve(256);
345  m_dir_names_stack.reserve(32);
347  }
static const Configuration & Conf()
Definition: XrdPfc.cc:162
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:160
const int m_max_dir_level_for_stat_collection
Definition: XrdPfcPurge.cc:263
std::vector< long long > m_dir_usage_stack
Definition: XrdPfcPurge.cc:266
const char * m_info_ext
Definition: XrdPfcPurge.cc:268
std::vector< std::string > m_dir_names_stack
Definition: XrdPfcPurge.cc:265
XrdSysTrace * m_trace
Definition: XrdPfcPurge.cc:270
const size_t m_info_ext_len
Definition: XrdPfcPurge.cc:269
std::string m_current_path
Definition: XrdPfcPurge.cc:261
DirState * m_dir_state
Definition: XrdPfcPurge.cc:260
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:313
XrdSysTrace * GetTrace()
Definition: XrdPfcPurge.cc:16

References m_current_path, m_dir_names_stack, m_dir_usage_stack, and m_max_dir_level_for_stat_collection.

Member Function Documentation

◆ begin_traversal()

void XrdPfc::FPurgeState::begin_traversal ( DirState root,
const char *  root_path = "/" 
)
inline

Definition at line 275 of file XrdPfcPurge.cc.

276  {
277  m_dir_state = root;
278  m_dir_level = 0;
279  m_current_path = std::string(root_path);
280  m_dir_usage_stack.push_back(0);
281 
282  TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", level=" << m_dir_level);
283  }
#define TRACE_PURGE(x)
Definition: XrdPfcPurge.cc:25

References m_current_path, m_dir_level, m_dir_state, m_dir_usage_stack, and TRACE_PURGE.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ cd_down()

void XrdPfc::FPurgeState::cd_down ( const std::string &  dir_name)
inline

Definition at line 294 of file XrdPfcPurge.cc.

295  {
296  ++m_dir_level;
297 
299  {
300  m_dir_usage_stack.push_back(0);
301  m_dir_state = m_dir_state->find_dir(dir_name, true);
302  }
303 
304  m_dir_names_stack.push_back(dir_name);
305  m_current_path.append(dir_name);
306  m_current_path.append("/");
307  }
DirState * find_dir(const std::string &dir, bool create_subdirs)
Definition: XrdPfcPurge.cc:114

References XrdPfc::DirState::find_dir(), m_current_path, m_dir_level, m_dir_names_stack, m_dir_state, m_dir_usage_stack, and m_max_dir_level_for_stat_collection.

Referenced by TraverseNamespace().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ cd_up()

void XrdPfc::FPurgeState::cd_up ( )
inline

Definition at line 309 of file XrdPfcPurge.cc.

310  {
312  {
313  long long tail = m_dir_usage_stack.back();
314  m_dir_usage_stack.pop_back();
315 
316  TRACE_PURGE("FPurgeState::cd_up reporting for '" << m_current_path << "', usage=" << tail << ", level=" << m_dir_level);
317 
318  m_dir_state->set_usage(tail);
320 
321  m_dir_usage_stack.back() += tail;
322  }
323 
324  // remove trailing / and last dir but keep the new trailing / in place.
325  m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1);
326  m_dir_names_stack.pop_back();
327 
328  --m_dir_level;
329  }
DirState * get_parent()
Definition: XrdPfcPurge.cc:101
void set_usage(long long u)
Definition: XrdPfcPurge.cc:103

References XrdPfc::DirState::get_parent(), m_current_path, m_dir_level, m_dir_names_stack, m_dir_state, m_dir_usage_stack, m_max_dir_level_for_stat_collection, XrdPfc::DirState::set_usage(), and TRACE_PURGE.

Referenced by TraverseNamespace().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ CheckFile()

void XrdPfc::FPurgeState::CheckFile ( const char *  fname,
Info info,
struct stat fstat 
)
inline

Definition at line 391 of file XrdPfcPurge.cc.

392  {
393  static const char *trc_pfx = "FPurgeState::CheckFile ";
394 
395  long long nbytes = info.GetNDownloadedBytes();
396  time_t atime;
397  if ( ! info.GetLatestDetachTime(atime))
398  {
399  // cinfo file does not contain any known accesses, use fstat.mtime instead.
400  TRACE(Debug, trc_pfx << "could not get access time for " << m_current_path << fname << ", using mtime from stat instead.");
401  atime = fstat.st_mtime;
402  }
403  // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime);
404 
405  nBytesTotal += nbytes;
406 
407  m_dir_usage_stack.back() += nbytes;
408 
409  // XXXX Should remove aged-out files here ... but I have trouble getting
410  // the DirState and purge report set up consistently.
411  // Need some serious code reorganization here.
412  // Biggest problem is maintaining overall state a traversal state consistently.
413  // Sigh.
414 
415  // In first two cases we lie about FS time (set to 0) to get them all removed early.
416  // The age-based purge atime would also be good as there should be nothing
417  // before that time in the map anyway.
418  // But we use 0 as a test in purge loop to make sure we continue even if enough
419  // disk-space has been freed.
420 
421  if (tMinTimeStamp > 0 && atime < tMinTimeStamp)
422  {
423  m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state));
424  nBytesAccum += nbytes;
425  }
426  else if (tMinUVKeepTimeStamp > 0 &&
427  Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) &&
429  {
430  m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state));
431  nBytesAccum += nbytes;
432  }
433  else if (nBytesAccum < nBytesReq || ( ! m_fmap.empty() && atime < m_fmap.rbegin()->first))
434  {
435  m_fmap.insert(std::make_pair(atime, FS(m_current_path, fname, nbytes, atime, m_dir_state)));
436  nBytesAccum += nbytes;
437 
438  // remove newest files from map if necessary
439  while ( ! m_fmap.empty() && nBytesAccum - m_fmap.rbegin()->second.nBytes >= nBytesReq)
440  {
441  nBytesAccum -= m_fmap.rbegin()->second.nBytes;
442  m_fmap.erase(--(m_fmap.rbegin().base()));
443  }
444  }
445  }
int fstat(int fildes, struct stat *buf)
#define TRACE(act, x)
Definition: XrdTrace.hh:63
time_t GetNoCkSumTimeForUVKeep() const
Definition: XrdPfcInfo.hh:305
CkSumCheck_e GetCkSumState() const
Definition: XrdPfcInfo.hh:290
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
Definition: XrdPfcInfo.cc:472
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:411

References XrdPfc::Cache::Conf(), Macaroons::Debug, fstat(), XrdPfc::Info::GetCkSumState(), XrdPfc::Info::GetLatestDetachTime(), XrdPfc::Info::GetNDownloadedBytes(), XrdPfc::Info::GetNoCkSumTimeForUVKeep(), m_current_path, m_dir_state, m_dir_usage_stack, m_flist, m_fmap, nBytesAccum, nBytesReq, nBytesTotal, tMinTimeStamp, tMinUVKeepTimeStamp, and TRACE.

Referenced by TraverseNamespace().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ end_traversal()

void XrdPfc::FPurgeState::end_traversal ( )
inline

Definition at line 285 of file XrdPfcPurge.cc.

286  {
287  TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", nBytesTotal=" << nBytesTotal << ", level=" << m_dir_level);
288 
290 
291  m_dir_state = 0;
292  }

References m_current_path, m_dir_level, m_dir_state, m_dir_usage_stack, nBytesTotal, XrdPfc::DirState::set_usage(), and TRACE_PURGE.

Referenced by XrdPfc::Cache::Purge().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMinTime()

time_t XrdPfc::FPurgeState::getMinTime ( ) const
inline

Definition at line 352 of file XrdPfcPurge.cc.

352 { return tMinTimeStamp; }

References tMinTimeStamp.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ getNBytesTotal()

long long XrdPfc::FPurgeState::getNBytesTotal ( ) const
inline

Definition at line 354 of file XrdPfcPurge.cc.

354 { return nBytesTotal; }

References nBytesTotal.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ MoveListEntriesToMap()

void XrdPfc::FPurgeState::MoveListEntriesToMap ( )
inline

Definition at line 356 of file XrdPfcPurge.cc.

357  {
358  for (list_i i = m_flist.begin(); i != m_flist.end(); ++i)
359  {
360  m_fmap.insert(std::make_pair(i->time, *i));
361  }
362  m_flist.clear();
363  }
list_t::iterator list_i
Definition: XrdPfcPurge.cc:243

References m_flist, and m_fmap.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ setMinTime()

void XrdPfc::FPurgeState::setMinTime ( time_t  min_time)
inline

Definition at line 351 of file XrdPfcPurge.cc.

351 { tMinTimeStamp = min_time; }

References tMinTimeStamp.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ setUVKeepMinTime()

void XrdPfc::FPurgeState::setUVKeepMinTime ( time_t  min_time)
inline

Definition at line 353 of file XrdPfcPurge.cc.

353 { tMinUVKeepTimeStamp = min_time; }

References tMinUVKeepTimeStamp.

Referenced by XrdPfc::Cache::Purge().

+ Here is the caller graph for this function:

◆ TraverseNamespace()

void XrdPfc::FPurgeState::TraverseNamespace ( XrdOssDF iOssDF)
inline

Definition at line 447 of file XrdPfcPurge.cc.

448  {
449  static const char *trc_pfx = "FPurgeState::TraverseNamespace ";
450 
451  char fname[256];
452  struct stat fstat;
453  XrdOucEnv env;
454 
455  TRACE_PURGE("Starting to read dir [" << m_current_path << "], iOssDF->getFD()=" << iOssDF->getFD() << ".");
456 
457  iOssDF->StatRet(&fstat);
458 
459  while (true)
460  {
461  int rc = iOssDF->Readdir(fname, 256);
462 
463  if (rc == -ENOENT) {
464  TRACE_PURGE(" Skipping ENOENT dir entry [" << fname << "].");
465  continue;
466  }
467  if (rc != XrdOssOK) {
468  TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << ".");
469  break;
470  }
471 
472  TRACE_PURGE(" Readdir [" << fname << "]");
473 
474  if (fname[0] == 0) {
475  TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop.");
476  break;
477  }
478  if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) {
479  TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop.");
480  continue;
481  }
482 
483  size_t fname_len = strlen(fname);
484  XrdOssDF *dfh = 0;
485 
486  if (S_ISDIR(fstat.st_mode))
487  {
488  if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK)
489  {
490  cd_down(fname); TRACE_PURGE(" cd_down -> [" << m_current_path << "].");
491  TraverseNamespace(dfh);
492  cd_up(); TRACE_PURGE(" cd_up -> [" << m_current_path << "].");
493  }
494  else
495  TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno));
496  }
497  else if (fname_len > m_info_ext_len && strncmp(&fname[fname_len - m_info_ext_len], m_info_ext, m_info_ext_len) == 0)
498  {
499  // Check if the file is currently opened / purge-protected is done before unlinking of the file.
500 
501  Info cinfo(m_trace);
502 
503  if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname))
504  {
505  CheckFile(fname, cinfo, fstat);
506  }
507  else
508  {
509  TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging.");
510  m_oss_at.Unlink(*iOssDF, fname);
511  fname[fname_len - m_info_ext_len] = 0;
512  m_oss_at.Unlink(*iOssDF, fname);
513  }
514  }
515  else // XXXX devel debug only, to be removed
516  {
517  TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo.");
518  }
519 
520  delete dfh;
521  }
522  }
#define XrdOssOK
Definition: XrdOss.hh:50
int stat(const char *path, struct stat *buf)
@ Warning
const char * XrdSysE2T(int errcode)
Definition: XrdSysE2T.cc:99
int OpenRO(XrdOssDF &atDir, const char *path, XrdOucEnv &env, XrdOssDF *&ossDF)
Definition: XrdOssAt.cc:127
int Opendir(XrdOssDF &atDir, const char *path, XrdOucEnv &env, XrdOssDF *&ossDF)
Definition: XrdOssAt.cc:96
int Unlink(XrdOssDF &atDir, const char *path)
Definition: XrdOssAt.cc:199
virtual int StatRet(struct stat *buff)
Definition: XrdOss.hh:107
virtual int Readdir(char *buff, int blen)
Definition: XrdOss.hh:92
virtual int getFD()
Definition: XrdOss.hh:426
void CheckFile(const char *fname, Info &info, struct stat &fstat)
Definition: XrdPfcPurge.cc:391
void cd_down(const std::string &dir_name)
Definition: XrdPfcPurge.cc:294
void TraverseNamespace(XrdOssDF *iOssDF)
Definition: XrdPfcPurge.cc:447
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:45

References cd_down(), cd_up(), CheckFile(), Macaroons::Error, fstat(), XrdOssDF::getFD(), m_current_path, m_info_ext, m_info_ext_len, m_oss_at, m_trace, XrdOssAt::Opendir(), XrdOssAt::OpenRO(), XrdPfc::Info::Read(), XrdOssDF::Readdir(), stat(), XrdOssDF::StatRet(), TRACE, TRACE_PURGE, XrdOssAt::Unlink(), Warning, XrdOssOK, and XrdSysE2T().

Referenced by XrdPfc::Cache::Purge().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Member Data Documentation

◆ m_current_path

std::string XrdPfc::FPurgeState::m_current_path

◆ m_dir_level

int XrdPfc::FPurgeState::m_dir_level

Definition at line 262 of file XrdPfcPurge.cc.

Referenced by begin_traversal(), cd_down(), cd_up(), and end_traversal().

◆ m_dir_names_stack

std::vector<std::string> XrdPfc::FPurgeState::m_dir_names_stack

Definition at line 265 of file XrdPfcPurge.cc.

Referenced by FPurgeState(), cd_down(), and cd_up().

◆ m_dir_state

DirState* XrdPfc::FPurgeState::m_dir_state

Definition at line 260 of file XrdPfcPurge.cc.

Referenced by begin_traversal(), cd_down(), cd_up(), CheckFile(), and end_traversal().

◆ m_dir_usage_stack

std::vector<long long> XrdPfc::FPurgeState::m_dir_usage_stack

Definition at line 266 of file XrdPfcPurge.cc.

Referenced by FPurgeState(), begin_traversal(), cd_down(), cd_up(), CheckFile(), and end_traversal().

◆ m_flist

list_t XrdPfc::FPurgeState::m_flist

Definition at line 245 of file XrdPfcPurge.cc.

Referenced by CheckFile(), and MoveListEntriesToMap().

◆ m_fmap

map_t XrdPfc::FPurgeState::m_fmap

Definition at line 240 of file XrdPfcPurge.cc.

Referenced by CheckFile(), MoveListEntriesToMap(), and XrdPfc::Cache::Purge().

◆ m_info_ext

const char* XrdPfc::FPurgeState::m_info_ext

Definition at line 268 of file XrdPfcPurge.cc.

Referenced by TraverseNamespace().

◆ m_info_ext_len

const size_t XrdPfc::FPurgeState::m_info_ext_len

Definition at line 269 of file XrdPfcPurge.cc.

Referenced by TraverseNamespace().

◆ m_max_dir_level_for_stat_collection

const int XrdPfc::FPurgeState::m_max_dir_level_for_stat_collection

Definition at line 263 of file XrdPfcPurge.cc.

Referenced by FPurgeState(), cd_down(), and cd_up().

◆ m_oss_at

XrdOssAt XrdPfc::FPurgeState::m_oss_at

Definition at line 254 of file XrdPfcPurge.cc.

Referenced by TraverseNamespace().

◆ m_trace

XrdSysTrace* XrdPfc::FPurgeState::m_trace

Definition at line 270 of file XrdPfcPurge.cc.

Referenced by TraverseNamespace().

◆ m_traceID

const char * XrdPfc::FPurgeState::m_traceID = "Purge"
static

Definition at line 272 of file XrdPfcPurge.cc.

◆ nBytesAccum

long long XrdPfc::FPurgeState::nBytesAccum

Definition at line 248 of file XrdPfcPurge.cc.

Referenced by CheckFile().

◆ nBytesReq

long long XrdPfc::FPurgeState::nBytesReq

Definition at line 247 of file XrdPfcPurge.cc.

Referenced by CheckFile().

◆ nBytesTotal

long long XrdPfc::FPurgeState::nBytesTotal

Definition at line 249 of file XrdPfcPurge.cc.

Referenced by CheckFile(), end_traversal(), and getNBytesTotal().

◆ tMinTimeStamp

time_t XrdPfc::FPurgeState::tMinTimeStamp

Definition at line 250 of file XrdPfcPurge.cc.

Referenced by CheckFile(), getMinTime(), and setMinTime().

◆ tMinUVKeepTimeStamp

time_t XrdPfc::FPurgeState::tMinUVKeepTimeStamp

Definition at line 251 of file XrdPfcPurge.cc.

Referenced by CheckFile(), and setUVKeepMinTime().


The documentation for this class was generated from the following file: