XRootD
XrdPfcFPurgeState.cc
Go to the documentation of this file.
1 #include "XrdPfcFPurgeState.hh"
2 #include "XrdPfcFsTraversal.hh"
3 #include "XrdPfcInfo.hh"
4 #include "XrdPfc.hh"
5 #include "XrdPfcTrace.hh"
6 
7 #include "XrdOuc/XrdOucEnv.hh"
8 #include "XrdOuc/XrdOucUtils.hh"
9 #include "XrdOss/XrdOss.hh"
10 #include "XrdOss/XrdOssAt.hh"
11 
12 // Temporary, extensive purge tracing
13 // #define TRACE_PURGE(x) TRACE(Debug, x)
14 // #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n"
15 #define TRACE_PURGE(x)
16 
17 using namespace XrdPfc;
18 
19 namespace
20 {
21  XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); }
22 }
23 
24 const char *FPurgeState::m_traceID = "Purge";
25 
26 //----------------------------------------------------------------------------
28 //----------------------------------------------------------------------------
29 FPurgeState::FPurgeState(long long iNBytesReq, XrdOss &oss) :
30  m_oss(oss),
31  m_nStBlocksReq((iNBytesReq >> 9) + 1ll), m_nStBlocksAccum(0), m_nStBlocksTotal(0),
32  m_tMinTimeStamp(0), m_tMinUVKeepTimeStamp(0)
33 {
34 
35 }
36 
37 //----------------------------------------------------------------------------
40 //----------------------------------------------------------------------------
42 {
43  for (list_i i = m_flist.begin(); i != m_flist.end(); ++i)
44  {
45  m_fmap.insert(std::make_pair(i->time, *i));
46  }
47  m_flist.clear();
48 }
49 
50 //----------------------------------------------------------------------------
57 //----------------------------------------------------------------------------
58 void FPurgeState::CheckFile(const FsTraversal &fst, const char *fname, Info &info, struct stat &fstat)
59 {
60  static const char *trc_pfx = "FPurgeState::CheckFile ";
61 
62  long long nblocks = fstat.st_blocks;
63  time_t atime;
64  if (!info.GetLatestDetachTime(atime))
65  {
66  // cinfo file does not contain any known accesses, use fstat.mtime instead.
67  TRACE(Debug, trc_pfx << "could not get access time for " << fst.m_current_path << fname << ", using mtime from stat instead.");
68  atime = fstat.st_mtime;
69  }
70  // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime);
71 
72  m_nStBlocksTotal += nblocks;
73 
74  // Could remove aged-out / uv-keep-failed files here ... or in the calling function that
75  // can aggreagate info for all files in the directory.
76 
77  // For now keep using 0 time as this is used in the purge loop to make sure we continue even if enough
78  // disk-space has been freed.
79 
80  if (m_tMinTimeStamp > 0 && atime < m_tMinTimeStamp)
81  {
82  m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nblocks, 0));
83  m_nStBlocksAccum += nblocks;
84  }
85  else if (m_tMinUVKeepTimeStamp > 0 &&
86  Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) &&
87  info.GetNoCkSumTimeForUVKeep() < m_tMinUVKeepTimeStamp)
88  {
89  m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nblocks, 0));
90  m_nStBlocksAccum += nblocks;
91  }
92  else if (m_nStBlocksAccum < m_nStBlocksReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first))
93  {
94  m_fmap.insert(std::make_pair(atime, PurgeCandidate(fst.m_current_path, fname, nblocks, atime)));
95  m_nStBlocksAccum += nblocks;
96 
97  // remove newest files from map if necessary
98  while (!m_fmap.empty() && m_nStBlocksAccum - m_fmap.rbegin()->second.nStBlocks >= m_nStBlocksReq)
99  {
100  m_nStBlocksAccum -= m_fmap.rbegin()->second.nStBlocks;
101  m_fmap.erase(--(m_fmap.rbegin().base()));
102  }
103  }
104 }
105 
107 {
108  static const char *trc_pfx = "FPurgeState::ProcessDirAndRecurse ";
109 
110  for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
111  {
112  // Check if the file is currently opened / purge-protected is done before unlinking of the file.
113  const std::string &f_name = it->first;
114  const std::string i_name = f_name + Info::s_infoExtension;
115 
116  XrdOssDF *fh = nullptr;
117  Info cinfo(GetTrace());
118 
119  // XXX Note, the initial scan now uses stat information only!
120 
121  if (! it->second.has_both()) {
122  // cinfo or data file is missing. What do we do? Erase?
123  // Should really be checked in some other "consistency" traversal.
124  continue;
125  }
126 
127  if (fst.open_at_ro(i_name.c_str(), fh) == XrdOssOK &&
128  cinfo.Read(fh, fst.m_current_path.c_str(), i_name.c_str()))
129  {
130  CheckFile(fst, i_name.c_str(), cinfo, it->second.stat_data);
131  }
132  else
133  {
134  TRACE(Warning, trc_pfx << "can't open or read " << fst.m_current_path << i_name << ", err " << XrdSysE2T(errno) << "; purging.");
135  fst.unlink_at(i_name.c_str());
136  fst.unlink_at(f_name.c_str());
137  // generate purge event or not? or just flag possible discrepancy?
138  // should this really be done in some other consistency-check traversal?
139  }
140  fst.close_delete(fh);
141 
142  // Protected top-directories are skipped.
143  }
144 
145  std::vector<std::string> dirs;
146  dirs.swap(fst.m_current_dirs);
147  for (auto &dname : dirs)
148  {
149  if (fst.cd_down(dname))
150  {
152  fst.cd_up();
153  }
154  }
155 }
156 
157 bool FPurgeState::TraverseNamespace(const char *root_path)
158 {
159  bool success_p = true;
160 
161  FsTraversal fst(m_oss);
162  fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N?
163  // Also ... this onoly applies to /, not any root_path
164  if (fst.begin_traversal(root_path))
165  {
167  }
168  else
169  {
170  // Fail startup, can't open /.
171  success_p = false;
172  }
173  fst.end_traversal();
174 
175  return success_p;
176 }
177 
178 /*
179 void FPurgeState::UnlinkInfoAndData(const char *fname, long long nblocks, XrdOssDF *iOssDF)
180 {
181  fname[fname_len - m_info_ext_len] = 0;
182  if (nblocks > 0)
183  {
184  if ( ! Cache.GetInstance().IsFileActiveOrPurgeProtected(dataPath))
185  {
186  m_n_purged++;
187  m_bytes_purged += nblocks;
188  } else
189  {
190  m_n_purge_protected++;
191  m_bytes_purge_protected += nblocks;
192  m_dir_state->add_usage_purged(nblocks);
193  // XXXX should also tweak other stuff?
194  fname[fname_len - m_info_ext_len] = '.';
195  return;
196  }
197  }
198  m_oss_at.Unlink(*iOssDF, fname);
199  fname[fname_len - m_info_ext_len] = '.';
200  m_oss_at.Unlink(*iOssDF, fname);
201 }
202 */
#define XrdOssOK
Definition: XrdOss.hh:50
int stat(const char *path, struct stat *buf)
int fstat(int fildes, struct stat *buf)
@ Warning
const char * XrdSysE2T(int errcode)
Definition: XrdSysE2T.cc:99
#define TRACE(act, x)
Definition: XrdTrace.hh:63
static const Configuration & Conf()
Definition: XrdPfc.cc:132
XrdSysTrace * GetTrace()
Definition: XrdPfc.hh:278
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:130
void CheckFile(const FsTraversal &fst, const char *fname, Info &info, struct stat &fstat)
void ProcessDirAndRecurse(FsTraversal &fst)
FPurgeState(long long iNBytesReq, XrdOss &oss)
Constructor.
bool TraverseNamespace(const char *root_path)
list_t::iterator list_i
std::vector< std::string > m_current_dirs
int open_at_ro(const char *fname, XrdOssDF *&ossDF)
int close_delete(XrdOssDF *&ossDF)
bool begin_traversal(DirState *root, const char *root_path)
int unlink_at(const char *fname)
std::set< std::string > m_protected_top_dirs
bool cd_down(const std::string &dir_name)
std::map< std::string, FilePairStat > m_current_files
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:309
time_t GetNoCkSumTimeForUVKeep() const
Definition: XrdPfcInfo.hh:301
CkSumCheck_e GetCkSumState() const
Definition: XrdPfcInfo.hh:286
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
Definition: XrdPfcInfo.cc:472
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
Definition: XrdPfcInfo.cc:296
Definition: XrdPfc.hh:41