RE: Dual core Athlons and unsynced TSCs

[john stultz <johnstul@us.ibm.com>]

On Fri, 2006-01-13 at 15:09 -0500, Steven Rostedt wrote:
> On Fri, 2006-01-13 at 13:55 -0500, Lee Revell wrote:
> 
> > > 
> > > If that's what you want to know?
> > 
> > I want to know if clock_gettime(CLOCK_MONOTONIC, *ts) is actually
> > guaranteed to be monotonic on these machines AKA do we break POSIX
> > compliance or not.
> 
> Nope it doesn't work :-(
> 
> I ran this test:
>  http://www.kihontech.com/tests/rt/monotonic.c
> 
> And got this:
> 
> $ ./monotonic
> main program pid=8477
> hello from thread 0!
> hello from thread 1!
> hello from thread 2!
> hello from thread 3!
> hello from thread 4!
> Failed! prev: 6279.925610302   current: 6279.874615349

Not sure, but I think your test is broken.

While it serializes the checking and exchange of current and last, it
doesn't serialize the actual calling of clock_gettime(). 

Thus you can get something like:

last=0
	A:				B:
1:	now = clock_gettime()
2:					now = clock_gettime()
3:					atomic {
						test(now, last) [2>0]
						last = now
					}
4:	atomic {
		test(now, last) [1>2]
		FAIL!
	}


Attached is a similar testcase that I've been using myself that avoids
this issue (although I just converted it from gettimeofday to
clock_gettime, so there may still be an issue). Let me know if you have
any comments on it.

thanks
-john

/* threadtest.c 
 *		by: john stultz (johnstul@us.ibm.com)
 *		(C) Copyright IBM 2004, 2005, 2006
 *		Licensed under the GPL
 */
#include <stdio.h>
#include <sys/time.h>
#include <pthread.h>


/* serializes shared list access */
pthread_mutex_t list_lock = PTHREAD_MUTEX_INITIALIZER;
/* serializes console output */
pthread_mutex_t print_lock = PTHREAD_MUTEX_INITIALIZER;


#define MAX_THREADS 128
#define LISTSIZE 128


int done = 0;

struct timespec global_list[LISTSIZE];
int listcount = 0;


void checklist(struct timespec* list, int size)
{
	int i,j;
	struct timespec *a,*b;

	static int dbgct = 0;
	if(!(dbgct++%7000)){
		printf(".");
		fflush(0);
	}

	/* scan the list */
	for(i=0; i < size-1; i++){
		a = &list[i];
		b = &list[i+1];
		
		/* look for any time inconsistencies */
		if((b->tv_sec <= a->tv_sec)&&
			(b->tv_nsec < a->tv_nsec)){

			/* flag other threads */
			done = 1;

			/*serialize printing to avoid junky output*/
			pthread_mutex_lock(&print_lock);

			/* dump the list */
			printf("\n");
			for(j=0; j< size; j++){
				if(j == i)
					printf("---------------\n");
				printf("%lu:%lu\n", list[j].tv_sec,list[j].tv_nsec);
				if(j == i+1)
					printf("---------------\n");
			}
			printf("TEST FAILED\n");

			pthread_mutex_unlock(&print_lock);


		}
	}
}

/* The Shared Thread shares a global list
 * that each thread fills while holding the lock.
 * This stresses clock syncronization across cpus. 
 */
void* shared_thread(void* arg)
{
	while(!done){
		/* protect the list */
		pthread_mutex_lock(&list_lock);
		
		/* see if we're ready to check the list */
		if(listcount >= LISTSIZE){
			checklist(global_list, LISTSIZE);
			listcount = 0;
		}
		clock_gettime(CLOCK_MONOTONIC, &global_list[listcount++]);
				
		pthread_mutex_unlock(&list_lock);
	}
}


/* Each independent thread fills in its own
 * list. This stresses clock_gettime() lock contention. 
 */
void* independent_thread(void* arg)
{
	struct timespec my_list[LISTSIZE];
	int count;

	while(!done){
		/* fill the list */
		for(count=0; count < LISTSIZE; count++)
			clock_gettime(CLOCK_MONOTONIC, &my_list[count]);
		checklist(my_list, LISTSIZE);
	}
}


int main(int argc, void** argv)
{
	int thread_count = 1, i;
	pthread_t pth[MAX_THREADS];
	void* ret;
	void* (*thread)(void*) = shared_thread;
	
	/* pull the thread count */
	if(argc > 1)
		thread_count = strtol(argv[1],0,0);
	if(thread_count > MAX_THREADS)
		thread_count = MAX_THREADS;

	/* check if we're running independent threads */	
	if(argc == 3 && !strncmp(argv[2],"indie", 5)){
		thread = independent_thread;
		printf("using independent threads\n");
	}
	
	
	system("date");
	printf("spawning %i threads\n", thread_count,0,0);

	/* spawn */
	for(i=0; i < thread_count; i++)
		pthread_create(&pth[i], 0, thread, 0);
	
	/* wait */
	for(i=0; i< thread_count; i++)
		pthread_join(pth[i],&ret);

	system("date");	
	
	/* die */
	return 0;
}