[OpenAFS-devel] PATCH: break up cache into reasonable subdirs for large cache sizes
Derek Atkins
warlord@MIT.EDU
07 Jul 2001 10:21:59 -0400
Here is the patch for afsd that breaks up the cache into multiple
subdirs of a more reasonable size. Using this patch I could create
a 3GB cache in under 3 minutes on a Linux ext2 FS. I also added the
afsd parameter '-files_per_dir' which can set the log(2) size of the
subdirs. The default is 11 (2048 files).
This patch is against the 1.0.4 release.
Enjoy.
-derek
--- src/afsd/afsd.c-orig Mon Apr 23 03:29:03 2001
+++ src/afsd/afsd.c Sat Jul 7 10:17:47 2001
@@ -48,6 +48,8 @@
* This option is now disabled.
* -logfile Place where to put the logfile (default in <cache>/etc/AFSLog.
* -waitclose make close calls always synchronous (slows em down, tho)
+ * -files_per_subdir [n] 2^n is the number of files per cache subdir.
+ * 11 is default (meaning 2048 files)
* -shutdown Shutdown afs daemons
*---------------------------------------------------------------------------*/
@@ -188,7 +190,7 @@
char fullpn_AFSLogFile[1024]; /*Full pathname of AFSLOGFILE*/
char fullpn_CacheInfo[1024]; /*Full pathname of CACHEINFO*/
char fullpn_VFile[1024]; /*Full pathname of data cache files*/
-char *vFileNumber; /*Ptr to the number part of above pathname*/
+char *vFilePtr; /*Ptr to the number part of above pathname*/
int sawCacheMountDir = 0; /* from cmd line */
int sawCacheBaseDir = 0;
int sawCacheBlocks = 0;
@@ -202,6 +204,8 @@
int createAndTrunc = O_CREAT | O_TRUNC; /*Create & truncate on open*/
int ownerRWmode = 0600; /*Read/write OK by owner*/
static int filesSet = 0; /*True if number of files explicitly set*/
+static int expFilesPerDir = 11;
+static int nFilesPerDir; /* # files per cache dir (2048) */
static int nDaemons = 2; /* Number of background daemons */
static int chunkSize = 0; /* 2^chunkSize bytes per chunk */
static int dCacheSize = 300; /* # of dcache entries */
@@ -225,6 +229,7 @@
#else
#define AFSD_INO_T afs_uint32
#endif
+char *cache_dir_list = NULL; /* Array of cache subdirs */
AFSD_INO_T *inode_for_V; /* Array of inodes for desired
* cache files */
int missing_DCacheFile = 1; /*Is the DCACHEFILE missing?*/
@@ -387,6 +392,7 @@
*
* Arguments:
* fname : Char ptr to the filename to parse.
+ * max : integer for the highest number to accept
*
* Returns:
* >= 0 iff the file is really a data cache file numbered from 0 to cacheFiles-1, or
@@ -399,21 +405,23 @@
* None.
*---------------------------------------------------------------------------*/
-int GetVFileNumber(fname)
+static int doGetXFileNumber(fname, filechar, maxNum)
char *fname;
+ char filechar;
+ int maxNum;
{
int computedVNumber; /*The computed file number we return*/
int filenameLen; /*Number of chars in filename*/
int currDigit; /*Current digit being processed*/
/*
- * The filename must have at least two characters, the first of which must be a ``V''
+ * The filename must have at least two characters, the first of which must be a ``filechar''
* and the second of which cannot be a zero unless the file is exactly two chars long.
*/
filenameLen = strlen(fname);
if (filenameLen < 2)
return(-1);
- if (fname[0] != 'V')
+ if (fname[0] != filechar)
return(-1);
if ((filenameLen > 2) && (fname[1] == '0'))
return(-1);
@@ -437,6 +445,21 @@
return(-1);
}
+int GetVFileNumber(fname, maxFile)
+ char *fname;
+ int maxFile;
+{
+ return doGetXFileNumber(fname, 'V', maxFile);
+}
+
+int GetDDirNumber(fname, maxDir)
+ char *fname;
+ int maxDir;
+{
+ return doGetXFileNumber(fname, 'D', maxDir);
+}
+
+
/*-----------------------------------------------------------------------------
* CreateCacheFile
*
@@ -446,6 +469,8 @@
*
* Arguments:
* fname : Full pathname of file to create.
+ * statp : A pointer to a stat buffer which, if NON-NULL, will be
+ * filled by fstat()
*
* Returns:
* 0 iff the file was created,
@@ -458,8 +483,38 @@
* As described.
*---------------------------------------------------------------------------*/
-int CreateCacheFile(fname)
+int CreateCacheSubDir (basename, dirNum)
+ char *basename;
+ int dirNum;
+{
+ static char rn[] = "CreateCacheSubDir"; /* Routine Name */
+ char dir[1024];
+ int ret;
+
+ /* Build the new cache subdirectory */
+ sprintf (dir, "%s/D%d", basename, dirNum);
+
+ if (afsd_verbose)
+ printf("%s: Creating cache subdir '%s'\n",
+ rn, dir);
+
+ if ((ret = mkdir(dir, 0700)) != 0) {
+ printf("%s: Can't create '%s', error return is %d (%d)\n",
+ rn, dir, ret, errno);
+ if (errno != EEXIST)
+ return (-1);
+ }
+
+ /* Mark this directory as created */
+ cache_dir_list[dirNum] = 1;
+
+ /* And return success */
+ return (0);
+}
+
+int CreateCacheFile(fname, statp)
char *fname;
+ struct stat *statp;
{
static char rn[] = "CreateCacheFile"; /*Routine name*/
int cfd; /*File descriptor to AFS cache file*/
@@ -474,6 +529,14 @@
rn, fname, cfd, errno);
return(-1);
}
+ if (statp != NULL) {
+ closeResult = fstat (cfd, statp);
+ if (closeResult) {
+ printf("%s: Can't stat newly-created AFS cache file '%s' (code %d)\n",
+ rn, fname, errno);
+ return(-1);
+ }
+ }
closeResult = close(cfd);
if (closeResult) {
printf("%s: Can't close newly-created AFS cache file '%s' (code %d)\n",
@@ -489,7 +552,7 @@
*
* Description:
* Sweep through the AFS cache directory, recording the inode number for
- * each valid data cache file there. Also, delete any file that doesn't beint32
+ * each valid data cache file there. Also, delete any file that doesn't belong
* in the cache directory during this sweep, and remember which of the other
* residents of this directory were seen. After the sweep, we create any data
* cache files that were missing.
@@ -512,10 +575,15 @@
* explained above.
*---------------------------------------------------------------------------*/
-int SweepAFSCache(vFilesFound)
- int *vFilesFound;
+
+static int doSweepAFSCache(vFilesFound,directory,dirNum,maxFile,maxDir)
+ int *vFilesFound;
+ char *directory; /* /path/to/cache/directory */
+ int dirNum; /* current directory number */
+ int maxFile; /* maximum file number for this directory */
+ int maxDir; /* maximum directory number */
{
- static char rn[] = "SweepAFSCache"; /*Routine name*/
+ static char rn[] = "doSweepAFSCache"; /* Routine Name */
char fullpn_FileToDelete[1024]; /*File to be deleted from cache*/
char *fileToDelete; /*Ptr to last component of above*/
DIR *cdirp; /*Ptr to cache directory structure*/
@@ -525,36 +593,33 @@
struct dirent *currp; /*Current directory entry*/
#endif
int vFileNum; /*Data cache file's associated number*/
-
- if (cacheFlags & AFSCALL_INIT_MEMCACHE) {
- if (afsd_debug)
- printf("%s: Memory Cache, no cache sweep done\n", rn);
- *vFilesFound = 0;
- return 0;
- }
+ int thisDir;
if (afsd_debug)
- printf("%s: Opening cache directory '%s'\n",
- rn, cacheBaseDir);
+ printf("%s: Opening cache directory '%s'\n", rn, directory);
- if (chmod(cacheBaseDir, 0700)) { /* force it to be 700 */
- printf("%s: Can't 'chmod 0700' the cache dir, '%s'.\n",
- rn, cacheBaseDir);
+ if (chmod(directory, 0700)) { /* force it to be 700 */
+ printf("%s: Can't 'chmod 0700' the cache dir, '%s'.\n", rn, directory);
return (-1);
}
- cdirp = opendir(cacheBaseDir);
+ cdirp = opendir(directory);
if (cdirp == (DIR *)0) {
- printf("%s: Can't open AFS cache directory, '%s'.\n",
- rn, cacheBaseDir);
+ printf("%s: Can't open AFS cache directory, '%s'.\n", rn, directory);
return(-1);
}
+ /* Precompute this directory number */
+ if (dirNum >= 0)
+ thisDir = dirNum << expFilesPerDir;
+
/*
- * Scan the directory entries, remembering data cache file inodes and the existance
- * of other important residents. Delete all files that don't belong here.
+ * Scan the directory entries, remembering data cache file inodes
+ * and the existance of other important residents. Recurse into
+ * the data subdirectories.
+ *
+ * Delete all files and directories that don't belong here.
*/
- *vFilesFound = 0;
- sprintf(fullpn_FileToDelete, "%s/", cacheBaseDir);
+ sprintf(fullpn_FileToDelete, "%s/", directory);
fileToDelete = fullpn_FileToDelete + strlen(fullpn_FileToDelete);
#ifdef AFS_SGI62_ENV
@@ -576,24 +641,48 @@
}
/*
- * Guess current entry is for a data cache file.
+ * If dirNum == -1, we are a top-level cache directory and should
+ * only contain sub-directories and other sundry files. Therefore,
+ * V-files are valid only if dirNum >= 0, and Directories are only
+ * valid if dirNum == -1.
*/
- vFileNum = GetVFileNumber(currp->d_name);
- if (vFileNum >= 0) {
+
+ if (dirNum >= 0 && (*(currp->d_name) == 'V') &&
+ ((vFileNum = GetVFileNumber(currp->d_name, maxFile)) >= 0)) {
/*
- * Found a valid data cache filename. Remember this file's inode and bump
- * the number of files found.
+ * Found a valid data cache filename. Remember this
+ * file's inode and bump the number of files found.
*/
- inode_for_V[vFileNum] = currp->d_ino;
+ inode_for_V[thisDir + vFileNum] = currp->d_ino;
(*vFilesFound)++;
}
- else if (strcmp(currp->d_name, DCACHEFILE) == 0) {
+ else if (dirNum == -1 && (*(currp->d_name) == 'D') &&
+ ((vFileNum = GetDDirNumber(currp->d_name, maxDir)) >= 0)) {
+ int retval;
+ int mf = min (nFilesPerDir,
+ (cacheFiles-(vFileNum << expFilesPerDir)));
+
+ /* Found a valid cachefile sub-Directory. Remember this number
+ * and recurse into it. Note that subdirs cannot have subdirs.
+ */
+ cache_dir_list[dirNum] = 1;
+
+ sprintf(fileToDelete, "%s", currp->d_name);
+ retval = doSweepAFSCache(vFilesFound, fullpn_FileToDelete,
+ vFileNum, mf, 0);
+ if (retval) {
+ printf ("%s: Recursive sweep failed on directory %s\n",
+ rn, currp->d_name);
+ return retval;
+ }
+ }
+ else if (dirNum == -1 && strcmp(currp->d_name, DCACHEFILE) == 0) {
/*
* Found the file holding the dcache entries.
*/
missing_DCacheFile = 0;
}
- else if (strcmp(currp->d_name, VOLINFOFILE) == 0) {
+ else if (dirNum == -1 && strcmp(currp->d_name, VOLINFOFILE) == 0) {
/*
* Found the file holding the volume info.
*/
@@ -614,55 +703,82 @@
}
else {
/*
- * This file doesn't belong in the cache. Nuke it.
+ * This file/directory doesn't belong in the cache. Nuke it.
*/
sprintf(fileToDelete, "%s", currp->d_name);
if (afsd_verbose)
printf("%s: Deleting '%s'\n",
rn, fullpn_FileToDelete);
if (unlink(fullpn_FileToDelete)) {
- printf("%s: Can't unlink '%s', errno is %d\n",
- rn, fullpn_FileToDelete, errno);
+ if (errno == EISDIR && *fileToDelete == 'D') {
+ if (rmdir(fullpn_FileToDelete)) {
+ printf("%s: Can't rmdir '%s', errno is %d\n",
+ rn, fullpn_FileToDelete, errno);
+ }
+ } else
+ printf("%s: Can't unlink '%s', errno is %d\n",
+ rn, fullpn_FileToDelete, errno);
}
}
}
- /*
- * Create all the cache files that are missing.
- */
- if (missing_DCacheFile) {
- if (afsd_verbose)
- printf("%s: Creating '%s'\n",
- rn, fullpn_DCacheFile);
- if (CreateCacheFile(fullpn_DCacheFile))
- printf("%s: Can't create '%s'\n",
- rn, fullpn_DCacheFile);
- }
- if (missing_VolInfoFile) {
- if (afsd_verbose)
- printf("%s: Creating '%s'\n",
- rn, fullpn_VolInfoFile);
- if (CreateCacheFile(fullpn_VolInfoFile))
- printf("%s: Can't create '%s'\n",
- rn, fullpn_VolInfoFile);
- }
+ if (dirNum == -1) {
- if (*vFilesFound < cacheFiles) {
- /*
- * We came up short on the number of data cache files found. Scan through the inode
- * list and create all missing files.
+ /*
+ * Create all the cache files that are missing.
*/
- for (vFileNum = 0; vFileNum < cacheFiles; vFileNum++)
- if (inode_for_V[vFileNum] == (AFSD_INO_T)0) {
- sprintf(vFileNumber, "%d", vFileNum);
- if (afsd_verbose)
- printf("%s: Creating '%s'\n",
- rn, fullpn_VFile);
- if (CreateCacheFile(fullpn_VFile))
- printf("%s: Can't create '%s'\n",
- rn, fullpn_VFile);
- }
- }
+ if (missing_DCacheFile) {
+ if (afsd_verbose)
+ printf("%s: Creating '%s'\n",
+ rn, fullpn_DCacheFile);
+ if (CreateCacheFile(fullpn_DCacheFile, NULL))
+ printf("%s: Can't create '%s'\n",
+ rn, fullpn_DCacheFile);
+ }
+ if (missing_VolInfoFile) {
+ if (afsd_verbose)
+ printf("%s: Creating '%s'\n",
+ rn, fullpn_VolInfoFile);
+ if (CreateCacheFile(fullpn_VolInfoFile, NULL))
+ printf("%s: Can't create '%s'\n",
+ rn, fullpn_VolInfoFile);
+ }
+
+ if (*vFilesFound < cacheFiles) {
+ /*
+ * We came up short on the number of data cache files found.
+ * Scan through the inode list and create all missing files.
+ */
+ for (thisDir = 0; thisDir < maxDir; thisDir++) {
+ int maxFile = min (nFilesPerDir,
+ (cacheFiles-(thisDir << expFilesPerDir)));
+ int dnum = thisDir << expFilesPerDir;
+
+ for (vFileNum = 0; vFileNum < maxFile; vFileNum++) {
+ int thisFile = dnum+vFileNum;
+
+ if (inode_for_V[thisFile] == (AFSD_INO_T)0) {
+ struct stat statb;
+
+ sprintf(vFilePtr, "D%d/V%d", thisDir, vFileNum);
+ if (afsd_verbose)
+ printf("%s: Creating '%s'\n", rn, fullpn_VFile);
+ if (cache_dir_list[thisDir] == 0 &&
+ CreateCacheSubDir(directory, thisDir))
+ printf("%s: Can't create directory for '%s'\n",
+ rn, fullpn_VFile);
+ if (CreateCacheFile(fullpn_VFile, &statb))
+ printf("%s: Can't create '%s'\n",
+ rn, fullpn_VFile);
+ else {
+ inode_for_V[thisFile] = statb.st_ino;
+ *vFilesFound++;
+ }
+ }
+ } /* for (fileNum...) */
+ } /* for (thisDir...) */
+ } /* filesfound < cachefiles */
+ } /* dirNum == -1 */
/*
* Close the directory, return success.
@@ -674,6 +790,33 @@
return(0);
}
+int SweepAFSCache(vFilesFound)
+ int *vFilesFound;
+{
+ static char rn[] = "SweepAFSCache"; /*Routine name*/
+ int maxDir = (cacheFiles >> expFilesPerDir) + 1;
+
+ *vFilesFound = 0;
+ nFilesPerDir = 1<<expFilesPerDir; /* pre-compute number of files */
+
+ if (cacheFlags & AFSCALL_INIT_MEMCACHE) {
+ if (afsd_debug)
+ printf("%s: Memory Cache, no cache sweep done\n", rn);
+ return 0;
+ }
+
+ if (cache_dir_list == NULL) {
+ cache_dir_list = (char *) malloc (maxDir);
+ if (cache_dir_list == NULL) {
+ printf("%s: Malloc Failed!\n", rn);
+ return (-1);
+ }
+ memset (cache_dir_list, 0, maxDir);
+ }
+
+ return doSweepAFSCache (vFilesFound, cacheBaseDir, -1, 0, maxDir);
+}
+
static ConfigCell(aci, arock, adir)
register struct afsconf_cell *aci;
char *arock;
@@ -896,6 +1039,14 @@
/* -mem_alloc_sleep */
cacheFlags |= AFSCALL_INIT_MEMCACHE_SLEEP;
}
+ if (as->parms[24].items) {
+ /* -files_per_subdir */
+ expFilesPerDir = atoi(as->parms[24].items->data);
+ if (expFilesPerDir < 2 || expFilesPerDir > 30) {
+ printf("afsd:invalid cache subdir size spec'd, forcing one subdir\n");
+ expFilesPerDir = 30;
+ }
+ }
/*
* Pull out all the configuration info for the workstation's AFS cache and
@@ -925,7 +1076,7 @@
if ((logfd = fopen(fullpn_AFSLogFile,"r+")) == 0) {
if (afsd_verbose) printf("%s: Creating '%s'\n", rn, fullpn_AFSLogFile);
- if (CreateCacheFile(fullpn_AFSLogFile)) {
+ if (CreateCacheFile(fullpn_AFSLogFile, NULL)) {
printf("%s: Can't create '%s' (You may want to use the -logfile option)\n", rn, fullpn_AFSLogFile);
exit(1);
}
@@ -1029,8 +1180,8 @@
*/
sprintf(fullpn_DCacheFile, "%s/%s", cacheBaseDir, DCACHEFILE);
sprintf(fullpn_VolInfoFile, "%s/%s", cacheBaseDir, VOLINFOFILE);
- sprintf(fullpn_VFile, "%s/V", cacheBaseDir);
- vFileNumber = fullpn_VFile + strlen(fullpn_VFile);
+ sprintf(fullpn_VFile, "%s/", cacheBaseDir);
+ vFilePtr = fullpn_VFile + strlen(fullpn_VFile);
#if 0
fputs(AFS_GOVERNMENT_MESSAGE, stdout);
@@ -1460,6 +1611,7 @@
cmd_AddParm(ts, "-enable_peer_stats", CMD_FLAG, CMD_OPTIONAL|CMD_HIDE, "Collect rpc statistics by peer");
cmd_AddParm(ts, "-enable_process_stats", CMD_FLAG, CMD_OPTIONAL|CMD_HIDE, "Collect rpc statistics for this process");
cmd_AddParm(ts, "-mem_alloc_sleep", CMD_FLAG, (CMD_OPTIONAL | CMD_HIDE), "Allow sleeps when allocating memory cache");
+ cmd_AddParm(ts, "-files_per_subdir", CMD_SINGLE, CMD_OPTIONAL, "log(2) of the number of cache files per cache subdirectory");
return (cmd_Dispatch(argc, argv));
}
--
Derek Atkins, SB '93 MIT EE, SM '95 MIT Media Laboratory
Member, MIT Student Information Processing Board (SIPB)
URL: http://web.mit.edu/warlord/ PP-ASEL-IA N1NWH
warlord@MIT.EDU PGP key available