[OpenAFS] Solaris 10u6: ZFS cache?
Douglas E. Engert
deengert@anl.gov
Wed, 12 Nov 2008 15:31:10 -0600
This is a multi-part message in MIME format.
--------------040207010305010907020401
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit
Derrick Brashear wrote:
>> The first patch in effect does not use inodes at all, all operations
>> on the cache files are done via vn_ or VOP_* operations using the the vnode.
>>
>> I tried creating a /tmp/afscache where /tmp in in swap. and it works!
>> The /etd/init.d/afs script needs to create the /tmp/afscache directory.
>>
>> So are you leaning to using the AFS_CACHE_VNODE_PATH approach
>> in all cases?
>
> We need to collectively decide it. But there's value in cache in tmpfs...
>
>> One of the trade offs is that I had to use the pn_get_buf, lookuppnvp
>> to get the vnode, so it could pass the afs_osi_cred.
>
> Does this mean future sadness?
I would say no. pn_get_buf, and lookuppnvp are defined in pathname.h
which are fs independent.
Spending most of today looking at how zfs sets the inode in stat and how
this could be used with the vfs_vget I can't pin it down. The ino_t is 64 bits,
but the fid_t has a 16 bit length, then 6 byte object and 4 byte generation.
There is also a longer fid.
The use of the AFS_CACHE_VNODE_PATH has the benafit that it should work with
any files system, as all access is done by vnode operations. This might be
a big improvment for other systems too, and it would simplify the kernel code.
Attached is a new patch using the AFS_CACHE_VNODE_PATH.
This fixes the if (ainode < 0) problem by removing it, and goes back to
using the 64 bit ainode. Thus all the code changes are in SOLARIS/osi_file.c
and the parm.<sysname>.h files.
If I were to speed it up, I would get the vnode for afs_cachebasedir
once, and then use it as the starting point for the calls to
lookuppnvp. I can do that if you want.
>
>> With the vfs_vget or tmp_vget approach one has to now how
>> to build the fid_t from the inode as returned by stat.
>>
>> I have been looking at the Open Solaris code for all this...
>>
>
>
--
Douglas E. Engert <DEEngert@anl.gov>
Argonne National Laboratory
9700 South Cass Avenue
Argonne, Illinois 60439
(630) 252-5444
--------------040207010305010907020401
Content-Type: text/plain;
name="Solaris-vnode-path-2.txt"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="Solaris-vnode-path-2.txt"
--- ./src/afs/SOLARIS/,osi_file.c Mon Mar 17 10:28:55 2008
+++ ./src/afs/SOLARIS/osi_file.c Wed Nov 12 14:49:46 2008
@@ -175,22 +175,85 @@
osi_UfsOpen(afs_int32 ainode)
#endif
{
+#ifdef AFS_CACHE_VNODE_PATH
+ struct vnode *vp;
+#else
struct inode *ip;
+#endif
register struct osi_file *afile = NULL;
afs_int32 code = 0;
+#ifdef AFS_CACHE_VNODE_PATH
int dummy;
+ char fname[1024];
+ char namebuf[1024];
+ struct pathname lookpn;
+#endif
+ struct osi_stat tstat;
afile = (struct osi_file *)osi_AllocSmallSpace(sizeof(struct osi_file));
AFS_GUNLOCK();
+
+/*
+ * AFS_CACHE_VNODE_PATH can be used with any file system, including ZFS or tmpfs.
+ * The ainode is not an inode number but a signed index used to generate file names.
+ */
+#ifdef AFS_CACHE_VNODE_PATH
+ switch (ainode) {
+ case AFS_CACHE_CELLS_INODE:
+ snprintf(fname, 1024, "%s/%s", afs_cachebasedir, "CellItems");
+ break;
+ case AFS_CACHE_ITEMS_INODE:
+ snprintf(fname, 1024, "%s/%s", afs_cachebasedir, "CacheItems");
+ break;
+ case AFS_CACHE_VOLUME_INODE:
+ snprintf(fname, 1024, "%s/%s", afs_cachebasedir, "VolumeItems");
+ break;
+ default:
+ dummy = ainode / afs_numfilesperdir;
+ snprintf(fname, 1024, "%s/D%d/V%d", afs_cachebasedir, dummy, ainode);
+ }
+
+ /* Can not use vn_open or lookupname, they use user's CRED()
+ * We need to run as root So must use low level lookuppnvp
+ * assume fname starts with /
+ */
+
+ code = pn_get_buf(fname, AFS_UIOSYS, &lookpn, namebuf, sizeof(namebuf));
+ if (code != 0)
+ osi_Panic("UfsOpen: pn_get_buf failed %ld %s %ld", code, fname, ainode);
+
+ VN_HOLD(rootdir); /* released in loopuppnvp */
+ code = lookuppnvp(&lookpn, NULL, FOLLOW, NULL, &vp,
+ rootdir, rootdir, &afs_osi_cred);
+ if (code != 0)
+ osi_Panic("UfsOpen: lookuppnvp failed %ld %s %ld", code, fname, ainode);
+
+#ifdef AFS_SUN511_ENV
+ code = VOP_OPEN(&vp, FREAD|FWRITE, &afs_osi_cred, NULL);
+#else
+ code = VOP_OPEN(&vp, FREAD|FWRITE, &afs_osi_cred);
+#endif
+
+ if (code != 0)
+ osi_Panic("UfsOpen: VOP_OPEN failed %ld %s %ld", code, fname, ainode);
+
+#else
code =
igetinode(afs_cacheVfsp, (dev_t) cacheDev.dev, (ino_t) ainode, &ip,
CRED(), &dummy);
+#endif
AFS_GLOCK();
if (code) {
osi_FreeSmallSpace(afile);
- osi_Panic("UfsOpen: igetinode failed");
+ osi_Panic("UfsOpen: igetinode failed %ld %s %ld", code, fname, ainode);
}
+#ifdef AFS_CACHE_VNODE_PATH
+ afile->vnode = vp;
+ code = afs_osi_Stat(afile, &tstat);
+ afile->size = tstat.size;
+#else
afile->vnode = ITOV(ip);
afile->size = VTOI(afile->vnode)->i_size;
+#endif
afile->offset = 0;
afile->proc = (int (*)())0;
afile->inum = ainode; /* for hint validity checking */
@@ -304,12 +367,14 @@
osi_DisableAtimes(struct vnode *avp)
{
if (afs_CacheFSType == AFS_SUN_UFS_CACHE) {
+#ifndef AFS_CACHE_VNODE_PATH
struct inode *ip = VTOI(avp);
rw_enter(&ip->i_contents, RW_READER);
mutex_enter(&ip->i_tlock);
ip->i_flag &= ~IACC;
mutex_exit(&ip->i_tlock);
rw_exit(&ip->i_contents);
+#endif
}
}
--- ./src/config/,param.sunx86_510.h Fri Dec 23 19:09:53 2005
+++ ./src/config/param.sunx86_510.h Fri Nov 7 08:45:47 2008
@@ -38,6 +38,8 @@
#define AFS_HAVE_FLOCK_SYSID 1
+#define AFS_CACHE_VNODE_PATH 1
+
#include <afs/afs_sysnames.h>
#define AFS_GLOBAL_SUNLOCK 1 /* For global locking */
--- ./src/config/,param.sunx86_511.h Thu Dec 28 15:59:45 2006
+++ ./src/config/param.sunx86_511.h Fri Nov 7 08:46:14 2008
@@ -39,6 +39,8 @@
#define AFS_HAVE_FLOCK_SYSID 1
+#define AFS_CACHE_VNODE_PATH 1
+
#include <afs/afs_sysnames.h>
#define AFS_GLOBAL_SUNLOCK 1 /* For global locking */
--- ./src/config/,param.sun4x_510.h Fri Dec 23 19:09:53 2005
+++ ./src/config/param.sun4x_510.h Fri Nov 7 08:43:53 2008
@@ -34,6 +34,8 @@
#define AFS_3DISPARES 1 /* Utilize the 3 available disk inode 'spares' */
#endif /* AFS_NAMEI_ENV */
+#define AFS_CACHE_VNODE_PATH 1
+
#include <afs/afs_sysnames.h>
#define AFS_GLOBAL_SUNLOCK 1 /* For global locking */
--- ./src/config/,param.sun4x_511.h Thu Dec 28 15:59:45 2006
+++ ./src/config/param.sun4x_511.h Fri Nov 7 08:45:10 2008
@@ -35,6 +35,8 @@
#define AFS_3DISPARES 1 /* Utilize the 3 available disk inode 'spares' */
#endif /* AFS_NAMEI_ENV */
+#define AFS_CACHE_VNODE_PATH 1
+
#include <afs/afs_sysnames.h>
#define AFS_GLOBAL_SUNLOCK 1 /* For global locking */
--------------040207010305010907020401--