[OpenAFS-devel] fileserver profiling

[Kyle Moffett]

On Mar 07, 2005, at 14:20, Derrick J Brashear wrote:
> The mmap() code presumably needs another thread, and would need to be
> different for lwp. Why have 2 sets of code, and use up another thread?

Err, how about this little C program.  It needs a read_memory_barrier() 
and
write_memory_barrier() definition for each architecture, but other than 
that,
it's pretty portable to most ANSI/POSIX-type systems, and reading it is
lock-free and syscall-free to boot.  You can use it anywhere you can 
get in
an "open(); mmap();" at program start, read memory, and use memory 
barriers,
in other words, basically everywhere.

The code should be pretty self explanatory, so enjoy :-D

Cheers,
Kyle Moffett

-----BEGIN GEEK CODE BLOCK-----
Version: 3.12
GCM/CS/IT/U d- s++: a18 C++++>$ UB/L/X/*++++(+)>$ P+++(++++)>$
L++++(+++) E W++(+) N+++(++) o? K? w--- O? M++ V? PS+() PE+(-) Y+
PGP+++ t+(+++) 5 X R? tv-(--) b++++(++) DI+ D+ G e->++++$ h!*()>++$ r  
!y?(-)
------END GEEK CODE BLOCK------


#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>

static void handler(int signal);

static void usage(const char *arg0, int err, const char *fmt, ...)
	__attribute__((__noreturn__));

int main(int argc, char **argv);

enum status_t {
	STATUS_RUNNING,
	STATUS_STOPPED,
	STATUS_RESTART,
};

static enum status_t status = STATUS_STOPPED;
static int last_signal = 0;

static void usage(const char *arg0, int err, const char *fmt, ...) {
	va_list ap;
	va_start(ap,fmt);
	if (fmt) {
		if (err) {
			char buf[41] = { };
			strerror_r(err, buf, 40);
			fprintf(stderr,"Error: %s: ",buf);
			err = 0;
		} else {
			fprintf(stderr,"Error: ");
		}
		vfprintf(stderr,fmt,ap);
		fprintf(stderr,"\n");
	}
	va_end(ap);
	
	if (err) {
		char buf[41] = { };
		strerror_r(err, buf, 40);
		fprintf(stderr,"Error: %s\n",buf);
		err = 0;
	}
	
	if (!arg0) arg0 = "[" __FILE__ "]";
	
	fprintf(stderr,
		"Usage: %s ( -h | <timefile> ) \n"
		"    -h:       Display this help text.\n"
		"    timefile: The name of the file in which to provide\n"
		"          shared access to a monotonic time via mmap.\n",
		arg0);
	exit(1);
}

static void handler (int signal) {
	last_signal = signal;
}

/* These are for PPC only; the read memory barrier does too much 
anyways */
#define read_memory_barrier()	__asm__ __volatile__ ("eieio": : 
:"memory")
#define write_memory_barrier()	__asm__ __volatile__ ("eieio": : 
:"memory")

/*
  * The "old_time" is the currently stored value. NOTE: This value is
  * designed to be read and written locklessly, assuming that reading
  * and writing a "long" is atomic on your platform:
  *   To read:
  *     do {
  *       sec1 = time[2];
  *       read_memory_barrier();
  *       usec = time[1];
  *       read_memory_barrier();
  *       sec2 = time[0];
  *       read_memory_barrier();
  *     } while(sec1 != sec2);
  *   To write:
  *     time[0] = sec;
  *     write_memory_barrier();
  *     time[1] = usec;
  *     write_memory_barrier();
  *     time[2] = sec;
  */
#if 0
struct timeval read_time_nolk(volatile long *time) {
	struct timeval res;
	long lastsecs;
	
	do {
		res.tv_sec = time[2];
		read_memory_barrier();
		res.tv_usec = time[1];
		read_memory_barrier();
		lastsecs = time[0];
		read_memory_barrier();
	} while (lastsecs != res.tv_sec);
	
	return res;
}
#endif

static inline void write_time_nolk(volatile long *time, struct timeval 
val) {
	time[0] = val.tv_sec;
	write_memory_barrier();
	time[1] = val.tv_usec;
	write_memory_barrier();
	time[2] = val.tv_sec;
	write_memory_barrier();
}

int main(int argc, char **argv) {
	int fd; void *mem;
	long nulltimebuf[3] = { 0, 0, 0 };
	
	volatile long *oldtime = NULL;
	struct timeval newtime = { 0, 0 };
	struct timeval zerotime = { 0, 0 };
	
	if (argc <= 0)
		usage(NULL,0,"Missing first argument!");
	
	if (argc != 2)
		usage(argv[0],0,"Invalid arguments!");
	
	if (strlen(argv[1]) == 0)
		usage(argv[0],0,"Invalid argument!");
	
	if (!strcmp(argv[1],"-h"))
		usage(argv[0],0,NULL);
	
	/* Trap most signals */
	signal(SIGHUP,&handler);
	signal(SIGINT,&handler);
	signal(SIGQUIT,&handler);
	signal(SIGTERM,&handler);
	
startup:
	/* Open the file */
	fd = open(argv[1], O_RDWR|O_CREAT|O_EXCL, 0666);
	if (fd < 0)
		usage(argv[0],errno,"Could not open file: '%s'",argv[1]);
	
	/* Extend the file */
	if (0 > write(fd, nulltimebuf, 3*sizeof(long)))
		usage(argv[0],errno,"Could not resize file: '%s'",argv[1]);
	
	/* Mmap the file */
	mem = mmap(NULL,4096,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0);
	if (mem == MAP_FAILED)
		usage(argv[0],errno,"Could not map file: '%s'",argv[1]);
	
	oldtime = (volatile long *)mem;
	write_time_nolk(oldtime,zerotime);
	
	/* Enter the run loop */
	status = STATUS_RUNNING;
	while(status == STATUS_RUNNING) {
		/* Get a new timestamp */
		int err = gettimeofday(&newtime,NULL);
		if (err) usage(argv[0],errno,"Could not get the time");
		
		/* Bound the microseconds within 10^6 */
		newtime.tv_usec %= 1000000;
		
		/* Check for the time going backwards.  Since we're the only
		 * writer, we can afford to ignore memory barriers here */
		if (oldtime[0] > newtime.tv_sec || (
					oldtime[0] == newtime.tv_sec &&
					oldtime[1] > newtime.tv_usec
				)) {
			/* Ahh, crud, just spew a warning */
			fprintf(stderr,"WARNING: Time regression: "
					"%lu seconds, %lu microseconds\n",
					oldtime[0] - newtime.tv_sec,
					oldtime[1] - newtime.tv_usec);
		} else {
			/* Ok, the time is fine, so store it in memory */
			write_time_nolk(oldtime,newtime);
		}
		
		/* Check our signal status, if we've received one, then we *
		 * need to handle it and restart or clean up and quit.     */
		switch(last_signal) {
		case 0:
			/* Since we've got no signal, sleep and repeat */
			usleep(1000);
			break;
			
		case SIGHUP:
			status = STATUS_RESTART;
			break;
			
		case SIGQUIT:
		case SIGINT:
		case SIGTERM:
			status = STATUS_STOPPED;
			break;
			
		default:
			fprintf(stderr,"WARNING: Unknown signal: %d\n",
					last_signal);
			status = STATUS_STOPPED;
			break;
		}
	}
	
	fprintf(stderr,"Caught signal, cleaning up...\n");
	
	/* First prevent new accesses */
	if (unlink(argv[1]))
		usage(argv[0],errno,"Could not delete file: '%s'",argv[1]);
	
	/* Now tell listening processes that we're stopped */
	write_time_nolk(oldtime,zerotime);
	
	if (munmap(mem,4096))
		usage(argv[0],errno,"Could not unmap file: '%s'",argv[1]);
	
	if (close(fd))
		usage(argv[0],errno,"Could not close file: '%s'",argv[1]);
	
	if (status == STATUS_RESTART) {
		fprintf(stderr,"Restarting...\n");
		goto startup;
	}
	
	fprintf(stderr,"Quitting...\n");
	exit(0);
}