XRootD
Loading...
Searching...
No Matches
XrdCmsCache.cc
Go to the documentation of this file.
1/******************************************************************************/
2/* */
3/* X r d C m s C a c h e . c c */
4/* */
5/* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */
6/* All Rights Reserved */
7/* Produced by Andrew Hanushevsky for Stanford University under contract */
8/* DE-AC02-76-SFO0515 with the Department of Energy */
9/* */
10/* This file is part of the XRootD software suite. */
11/* */
12/* XRootD is free software: you can redistribute it and/or modify it under */
13/* the terms of the GNU Lesser General Public License as published by the */
14/* Free Software Foundation, either version 3 of the License, or (at your */
15/* option) any later version. */
16/* */
17/* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20/* License for more details. */
21/* */
22/* You should have received a copy of the GNU Lesser General Public License */
23/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24/* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25/* */
26/* The copyright holder's institutional names and contributor's names may not */
27/* be used to endorse or promote products derived from this software without */
28/* specific prior written permission of the institution or contributor. */
29/******************************************************************************/
30
31#include <cstdio>
32#include <sys/types.h>
33
34#include "XrdCms/XrdCmsCache.hh"
35#include "XrdCms/XrdCmsRRQ.hh"
36#include "XrdCms/XrdCmsTrace.hh"
37
38#include "XrdSys/XrdSysTimer.hh"
39
40#include "Xrd/XrdJob.hh"
41#include "Xrd/XrdScheduler.hh"
42
43namespace XrdCms
44{
45extern XrdScheduler *Sched;
46}
47
48using namespace XrdCms;
49
50/******************************************************************************/
51/* G l o b a l s */
52/******************************************************************************/
53
55
56/******************************************************************************/
57/* L o c a l C l a s s e s */
58/******************************************************************************/
59
61{
62public:
63
64void DoIt() {Cache.Recycle(myList); delete this;}
65
67 : XrdJob("cache scrubber"), myList(List) {}
69
70private:
71
72XrdCmsKeyItem *myList;
73};
74
75/******************************************************************************/
76/* E x t e r n a l T h r e a d I n t e r f a c e s */
77/******************************************************************************/
78
79void *XrdCmsStartTickTock(void *carg)
80 {XrdCmsCache *myCache = (XrdCmsCache *)carg;
81 return myCache->TickTock();
82 }
83
84/******************************************************************************/
85/* P u b l i c C a c h e M a n i p u l a t i o n M e t h o d s */
86/******************************************************************************/
87/******************************************************************************/
88/* Public A d d F i l e */
89/******************************************************************************/
90
91// This method insert or updates information about a path in the cache.
92
93// Key was found: Location information is updated depending on mask
94// mask == 0 Indicates that the information is being refreshed.
95// Location information is nullified. The update deadline is set
96// QDelay seconds in the future. The entry window is set to the
97// current window to be held for a full fxhold period.
98// mask != 0 Indicates that some location information is now known.
99// Location information is updated according to the mask.
100// For a r/w location, the deadline is satisfied and all
101// callbacks are dispatched. For an r/o location the deadline
102// is satisfied if no r/w callback is pending. Any r/o
103// callback is dispatched. The Info object is ignored.
104
105// Key not found: A selective addition occurs, depending on Sel.Opts
106// Opts !Advisory: The entry is added to the cache with location information
107// set as passed (usually 0). The update deadline is us set to
108// DLTtime seconds in the future. The entry window is set
109// to the current window.
110// Opts Advisory: The call is ignored since we do not keep information about
111// paths that were never asked for.
112
113// Returns True If this is the first time location information was added
114// to the entry.
115// Returns False Otherwise.
116
118{
119 XrdCmsKeyItem *iP;
120 SMask_t xmask;
121 int isrw = (Sel.Opts & XrdCmsSelect::Write), isnew = 0;
122
123// Serialize processing
124//
125 myMutex.Lock();
126
127// Check for fast path processing
128//
129 if ( !(iP = Sel.Path.TODRef) || !(iP->Key.Equiv(Sel.Path)))
130 if ((iP = Sel.Path.TODRef = CTable.Find(Sel.Path)))
131 Sel.Path.Ref = iP->Key.Ref;
132
133// Add/Modify the entry
134//
135 if (iP)
136 {if (!mask)
137 {iP->Loc.deadline = QDelay + time(0);
138 iP->Loc.lifeline = nilTMO + iP->Loc.deadline;
139 iP->Loc.hfvec = 0; iP->Loc.pfvec = 0; iP->Loc.qfvec = 0;
140 iP->Loc.TOD_B = BClock;
141 iP->Key.TOD = Tock;
142 } else {
143 xmask = iP->Loc.pfvec;
144 if (Sel.Opts & XrdCmsSelect::Pending) iP->Loc.pfvec |= mask;
145 else iP->Loc.pfvec &= ~mask;
146 isnew = (iP->Loc.hfvec == 0) || (iP->Loc.pfvec != xmask);
147 iP->Loc.hfvec |= mask;
148 iP->Loc.qfvec &= ~mask;
149 if (isrw) {iP->Loc.deadline = 0;
150 if (iP->Loc.roPend || iP->Loc.rwPend)
151 Dispatch(Sel, iP, iP->Loc.roPend, iP->Loc.rwPend);
152 }
153 else {if (!iP->Loc.rwPend) iP->Loc.deadline = 0;
154 if (iP->Loc.roPend) Dispatch(Sel, iP, iP->Loc.roPend, 0);
155 }
156 }
157 } else if (!(Sel.Opts & XrdCmsSelect::Advisory))
158 {Sel.Path.TOD = Tock;
159 if ((iP = CTable.Add(Sel.Path)))
160 {iP->Loc.pfvec = (Sel.Opts&XrdCmsSelect::Pending?mask:0);
161 iP->Loc.hfvec = mask;
162 iP->Loc.TOD_B = BClock;
163 iP->Loc.qfvec = 0;
164 iP->Loc.deadline = QDelay + time(0);
165 iP->Loc.lifeline = nilTMO + iP->Loc.deadline;
166 Sel.Path.Ref = iP->Key.Ref;
167 Sel.Path.TODRef = iP; isnew = 1;
168 }
169 }
170
171// All done
172//
173 myMutex.UnLock();
174 return isnew;
175}
176
177/******************************************************************************/
178/* Public D e l F i l e */
179/******************************************************************************/
180
181// This method removes location information from existing valid entries. If an
182// existing valid entry is found, based on Sel.Opts the following occurs:
183
184// Opts Advisory only locate information is removed.
185// Opts !Advisory if the entry has no location information it is removed from
186// the cache, if possible.
187
188// TRUE is returned if the entry was valid but location information was cleared.
189// Otherwise, FALSE is returned.
190
192{
193 XrdCmsKeyItem *iP;
194 int gone4good;
195
196// Lock the hash table
197//
198 myMutex.Lock();
199
200// Look up the entry and remove server
201//
202 if ((iP = CTable.Find(Sel.Path)))
203 {iP->Loc.hfvec &= ~mask;
204 iP->Loc.pfvec &= ~mask;
205 if ((gone4good = (iP->Loc.hfvec == 0)))
206 {if (nilTMO) iP->Loc.lifeline = nilTMO + time(0);
207 if (!(Sel.Opts & XrdCmsSelect::Advisory)
208 && XrdCmsKeyItem::Unload(iP) && !CTable.Recycle(iP))
209 Say.Emsg("DelFile", "Delete failed for", iP->Key.Val);
210 }
211 } else gone4good = 0;
212
213// All done
214//
215 myMutex.UnLock();
216 return gone4good;
217}
218
219/******************************************************************************/
220/* Public G e t F i l e */
221/******************************************************************************/
222
223// This method looks up entries in the cache. An "entry not found" condition
224// holds is the entry was found but is marked as deleted.
225
226// Entry was found: Location information is passed bask. If the update deadline
227// has passed, it is nullified and 1 is returned. Otherwise,
228// -1 is returned indicating a query is in progress.
229
230// Entry not found: FALSE is returned.
231
233{
234 XrdCmsKeyItem *iP;
235 SMask_t bVec;
236 int retc;
237
238// Lock the hash table
239//
240 myMutex.Lock();
241
242// Look up the entry and return location information
243//
244 if ((iP = CTable.Find(Sel.Path)))
245 {if ((bVec = (iP->Loc.TOD_B < BClock
246 ? getBVec(iP->Key.TOD, iP->Loc.TOD_B) & mask : 0)))
247 {iP->Loc.hfvec &= ~bVec;
248 iP->Loc.pfvec &= ~bVec;
249 iP->Loc.qfvec &= ~mask;
250 iP->Loc.deadline = QDelay + time(0);
251 iP->Loc.lifeline = nilTMO + iP->Loc.deadline;
252 retc = -1;
253 } else if (iP->Loc.deadline)
254 if (iP->Loc.deadline > time(0)) retc = -1;
255 else {iP->Loc.deadline = 0; retc = 1;}
256 else retc = 1;
257
258 if (nilTMO && retc == 1 && iP->Loc.hfvec == 0
259 && iP->Loc.lifeline <= time(0)) retc = 0;
260
261 Sel.Vec.hf = okVec & iP->Loc.hfvec;
262 Sel.Vec.pf = okVec & iP->Loc.pfvec;
263 Sel.Vec.bf = okVec & (bVec | iP->Loc.qfvec); iP->Loc.qfvec = 0;
264 Sel.Path.Ref = iP->Key.Ref;
265 } else retc = 0;
266
267// All done
268//
269 myMutex.UnLock();
270 Sel.Path.TODRef = iP;
271 return retc;
272}
273
274/******************************************************************************/
275/* Public U n k F i l e */
276/******************************************************************************/
277
279{
280 EPNAME("UnkFile");
281 XrdCmsKeyItem *iP;
282
283// Make sure we have the proper information. If so, lock the hash table
284//
285 myMutex.Lock();
286
287// Look up the entry and if valid update the unqueried vector. Note that
288// this method may only be called after GetFile() or AddFile() for a new entry
289//
290 if ((iP = Sel.Path.TODRef))
291 {if (iP->Key.Equiv(Sel.Path)) iP->Loc.qfvec = mask;
292 else iP = 0;
293 }
294
295// Return result
296//
297 myMutex.UnLock();
298 DEBUG("rc=" <<(iP ? 1 : 0) <<" path=" <<Sel.Path.Val);
299 return (iP ? 1 : 0);
300}
301
302/******************************************************************************/
303/* Public W T 4 F i l e */
304/******************************************************************************/
305
307{
308 EPNAME("WT4File");
309 XrdCmsKeyItem *iP;
310 time_t Now;
311 int retc;
312
313// Make sure we have the proper information. If so, lock the hash table
314//
315 if (!Sel.InfoP) return DLTime;
316 myMutex.Lock();
317
318// Look up the entry and if valid add it to the callback queue. Note that
319// this method may only be called after GetFile() or AddFile() for a new entry
320//
321 if (!(iP = Sel.Path.TODRef) || !(iP->Key.Equiv(Sel.Path))) retc = DLTime;
322 else if (iP->Loc.hfvec != mask) retc = 1;
323 else {Now = time(0); retc = 0;
324 if (iP->Loc.deadline && iP->Loc.deadline <= Now)
325 iP->Loc.deadline = DLTime + Now;
326 Add2Q(Sel.InfoP, iP, Sel.Opts);
327 }
328
329// Return result
330//
331 myMutex.UnLock();
332 DEBUG("rc=" <<retc <<" path=" <<Sel.Path.Val);
333 return retc;
334}
335
336/******************************************************************************/
337/* P u b l i c A d m i n i s t r a t i v e C l a s s e s */
338/******************************************************************************/
339/******************************************************************************/
340/* public B o u n c e */
341/******************************************************************************/
342
343void XrdCmsCache::Bounce(SMask_t smask, int SNum)
344{
345
346// Simply indicate that this server bounced
347//
348 myMutex.Lock();
349 Bounced[SNum] = ++BClock;
350 okVec |= smask;
351 if (SNum > vecHi) vecHi = SNum;
352 myMutex.UnLock();
353}
354
355/******************************************************************************/
356/* Public D r o p */
357/******************************************************************************/
358
359void XrdCmsCache::Drop(SMask_t smask, int SNum, int xHi)
360{
361 SMask_t nmask(~smask);
362
363// Remove the node from the path list
364//
365 Paths.Remove(smask);
366
367// Remove the node from the list of valid nodes
368//
369 myMutex.Lock();
370 Bounced[SNum] = 0;
371 okVec &= nmask;
372 vecHi = xHi;
373 myMutex.UnLock();
374}
375
376/******************************************************************************/
377/* public I n i t */
378/******************************************************************************/
379
380int XrdCmsCache::Init(int fxHold, int fxDelay, int fxQuery, int seFS, int nxHold)
381{
382 XrdCmsKeyItem *iP;
383 pthread_t tid;
384
385// Indicate whether we are a shared-everything setup as this changes how we
386// dispatch clients to newly discovered files (see Dispatch()).
387//
388 isDFS = seFS;
389
390// Initialize the delay time and the bounce clock tick window
391//
392 DLTime = fxDelay; QDelay = fxQuery;
393 if (!(Tick = fxHold/XrdCmsKeyItem::TickRate)) Tick = 1;
394
395// Set the timeout for nil entries if one needs to be set. Since this may cause
396// an infinite lookup delay, adjust it to be no less than 10 minutes longer
397// than the overall deadline for lookups (QDelay/fxQuery).
398//
399 if (nxHold)
400 {if (nxHold < fxQuery+min_nxTime) nxHold = fxQuery+min_nxTime;
401 nilTMO = static_cast<unsigned int>(nxHold);
402 }
403
404// Start the clock thread
405//
406 if (XrdSysThread::Run(&tid, XrdCmsStartTickTock, (void *)this,
407 0, "Cache Clock"))
408 {Say.Emsg("Init", errno, "start cache clock");
409 return 0;
410 }
411
412// Get the first reserve of cache items
413//
414 iP = XrdCmsKeyItem::Alloc(0);
415 XrdCmsKeyItem::Unload((unsigned int)0);
416 iP->Recycle();
417
418// All done
419//
420 return 1;
421}
422
423/******************************************************************************/
424/* public T i c k T o c k */
425/******************************************************************************/
426
428{
429 XrdCmsKeyItem *iP;
430
431// Simply adjust the clock and trim old entries
432//
433 do {XrdSysTimer::Snooze(Tick);
434 myMutex.Lock();
435 Tock = (Tock+1) & XrdCmsKeyItem::TickMask;
436 Bhistory[Tock].Start = Bhistory[Tock].End = 0;
437 iP = XrdCmsKeyItem::Unload(Tock);
438 myMutex.UnLock();
439 if (iP) Sched->Schedule((XrdJob *)new XrdCmsCacheJob(iP));
440 } while(1);
441
442// Keep compiler happy
443//
444 return (void *)0;
445}
446
447/******************************************************************************/
448/* P r i v a t e M e t h o d s */
449/******************************************************************************/
450/******************************************************************************/
451/* A d d 2 Q */
452/******************************************************************************/
453
454void XrdCmsCache::Add2Q(XrdCmsRRQInfo *Info, XrdCmsKeyItem *iP, int selOpts)
455{
456 bool isrw = (selOpts & XrdCmsSelect::Write) != 0;
457 short Slot = (isrw ? iP->Loc.rwPend : iP->Loc.roPend);
458
459// Add the request to the appropriate pending queue
460//
461 Info->Key = iP;
462 Info->isRW= isrw;
463 Info->ifOP= (selOpts & XrdCmsSelect::ifWant);
464 if (!(Slot = RRQ.Add(Slot, Info))) Info->Key = 0;
465 else if (isrw) iP->Loc.rwPend = Slot;
466 else iP->Loc.roPend = Slot;
467}
468
469/******************************************************************************/
470/* D i s p a t c h */
471/******************************************************************************/
472
473void XrdCmsCache::Dispatch(XrdCmsSelect &Sel, XrdCmsKeyItem *iP,
474 short roQ, short rwQ)
475{
476
477// Dispatching shared-everything nodes is very different from shared-nothing
478// since one ready node means all are ready and we can use any one of them.
479// The current list of nodes must is provided by the caller adding the entry.
480// Note that the minimum number of nodes will always be set to 0 via config.
481//
482 if (isDFS)
483 {if (roQ && RRQ.Ready(roQ, iP, Sel.Vec.hf, Sel.Vec.pf & Sel.Vec.hf))
484 iP->Loc.roPend = 0;
485 if((rwQ && Sel.Vec.wf)
486 && RRQ.Ready(rwQ, iP, Sel.Vec.wf, Sel.Vec.pf & Sel.Vec.wf))
487 iP->Loc.rwPend = 0;
488 return;
489 }
490
491// Disptaching shared-nothing nodes is a one-shot affair. Only one node becomes
492// ready at a time and we can immediately disptach that node unless we need to
493// wait for more nodes to respond.
494//
495 if (roQ && RRQ.Ready(roQ, iP, iP->Loc.hfvec, iP->Loc.pfvec))
496 iP->Loc.roPend = 0;
497 if (rwQ && RRQ.Ready(rwQ, iP, iP->Loc.hfvec, iP->Loc.pfvec))
498 iP->Loc.rwPend = 0;
499}
500
501/******************************************************************************/
502/* g e t B V e c */
503/******************************************************************************/
504
505SMask_t XrdCmsCache::getBVec(unsigned int TODa, unsigned int &TODb)
506{
507 EPNAME("getBVec");
508 SMask_t BVec(0);
509 long long i;
510
511// See if we can use a previously calculated bVec
512//
513 if (Bhistory[TODa].End == BClock && Bhistory[TODa].Start <= TODb)
514 {Bhits++; TODb = BClock; return Bhistory[TODa].Vec;}
515
516// Calculate the new vector
517//
518 for (i = 0; i <= vecHi; i++)
519 if (TODb < Bounced[i]) BVec |= 1ULL << i;
520
521 Bhistory[TODa].Vec = BVec;
522 Bhistory[TODa].Start = TODb;
523 Bhistory[TODa].End = BClock;
524 TODb = BClock;
525 Bmiss++;
526 if (!(Bmiss & 0xff)) DEBUG("hits=" <<Bhits <<" miss=" <<Bmiss);
527 return BVec;
528}
529
530/******************************************************************************/
531/* R e c y c l e */
532/******************************************************************************/
533
534void XrdCmsCache::Recycle(XrdCmsKeyItem *theList)
535{
536 XrdCmsKeyItem *iP;
537 char msgBuff[100];
538 int numNull, numHave, numFree, numRecycled = 0;
539
540// Recycle the list of cache items, as needed
541//
542 while((iP = theList))
543 {theList = iP->Key.TODRef;
544 if (iP->Loc.roPend) RRQ.Del(iP->Loc.roPend, iP);
545 if (iP->Loc.rwPend) RRQ.Del(iP->Loc.rwPend, iP);
546 myMutex.Lock(); CTable.Recycle(iP); myMutex.UnLock();
547 numRecycled++;
548 }
549
550// See if we have enough items in reserve
551//
552 myMutex.Lock();
553 XrdCmsKeyItem::Stats(numHave, numFree, numNull);
554 if (numFree < XrdCmsKeyItem::minFree)
555 {myMutex.UnLock();
556 if (!(numNull /= 4)) numNull = 1;
557 numHave += XrdCmsKeyItem::minAlloc * numNull;
558 while(numNull--)
559 {myMutex.Lock();
560 numFree = XrdCmsKeyItem::Replenish();
561 myMutex.UnLock();
562 }
563 } else myMutex.UnLock();
564
565// Log the stats
566//
567 sprintf(msgBuff, "%d cache items; %d allocated %d free",
568 numRecycled, numHave, numFree);
569 Say.Emsg("Recycle", msgBuff);
570}
#define DEBUG(x)
#define EPNAME(x)
void * XrdCmsStartTickTock(void *carg)
unsigned long long SMask_t
XrdCmsCacheJob(XrdCmsKeyItem *List)
void Bounce(SMask_t smask, int SNum)
int GetFile(XrdCmsSelect &Sel, SMask_t mask)
int DelFile(XrdCmsSelect &Sel, SMask_t mask)
int AddFile(XrdCmsSelect &Sel, SMask_t mask)
friend class XrdCmsCacheJob
XrdCmsPList_Anchor Paths
int UnkFile(XrdCmsSelect &Sel, SMask_t mask)
void Drop(SMask_t mask, int SNum, int xHi)
int WT4File(XrdCmsSelect &Sel, SMask_t mask)
int Init(int fxHold, int fxDelay, int fxQuery, int seFS, int nxHold)
static const int min_nxTime
void * TickTock()
static XrdCmsKeyItem * Unload(unsigned int theTock)
Definition XrdCmsKey.cc:172
static const unsigned int TickMask
Definition XrdCmsKey.hh:151
static XrdCmsKeyItem * Alloc(unsigned int theTock)
Definition XrdCmsKey.cc:71
static const int minAlloc
Definition XrdCmsKey.hh:152
static int Replenish()
Definition XrdCmsKey.cc:131
XrdCmsKeyLoc Loc
Definition XrdCmsKey.hh:129
static const int minFree
Definition XrdCmsKey.hh:153
XrdCmsKey Key
Definition XrdCmsKey.hh:130
static const unsigned int TickRate
Definition XrdCmsKey.hh:150
static void Stats(int &isAlloc, int &isFree, int &wasEmpty)
Definition XrdCmsKey.cc:159
SMask_t hfvec
Definition XrdCmsKey.hh:92
SMask_t qfvec
Definition XrdCmsKey.hh:94
unsigned int TOD_B
Definition XrdCmsKey.hh:95
SMask_t pfvec
Definition XrdCmsKey.hh:93
int Equiv(XrdCmsKey &oth)
Definition XrdCmsKey.hh:60
XrdCmsKeyItem * TODRef
Definition XrdCmsKey.hh:51
unsigned char TOD
Definition XrdCmsKey.hh:55
char * Val
Definition XrdCmsKey.hh:52
unsigned char Ref
Definition XrdCmsKey.hh:56
XrdCmsKeyItem * Add(XrdCmsKey &Key)
Definition XrdCmsNash.cc:54
XrdCmsKeyItem * Find(XrdCmsKey &Key)
int Recycle(XrdCmsKeyItem *rip)
void Remove(SMask_t mask)
short Add(short Snum, XrdCmsRRQInfo *ip)
Definition XrdCmsRRQ.cc:76
void Del(short Snum, const void *Key)
Definition XrdCmsRRQ.cc:116
int Ready(int Snum, const void *Key, SMask_t mask1, SMask_t mask2)
Definition XrdCmsRRQ.cc:197
struct XrdCmsSelect::@92 Vec
XrdCmsRRQInfo * InfoP
XrdCmsKey Path
void Schedule(XrdJob *jp)
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)
static void Snooze(int seconds)
XrdCmsRRQ RRQ
Definition XrdCmsRRQ.cc:55
XrdCmsCache Cache
XrdScheduler * Sched
XrdSysError Say