/*
 * $Id: rawpkthreads.c,v 1.6 2000/12/18 00:08:37 alderman Exp $
 *
  This code uses threads and lets the system schedule them as best
  it can.  The result is a maximum of 14 concurrent streams with
  SCHED_RR and PTHREAD_SCOPE_USER, 13 with SCHED_FIFO, 12 with
  PTHREAD_SCOPE_SYSTEM
 */
// #define DEBUG
// #define DEBUGVERBOSE

#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/time.h>
#include <unistd.h>
#include <time.h>
#include <math.h>

#include "global.h"
#include "options.h"
#include "timer.h"
#include "since.h"

struct streamdata {             /* data structure for each stream / thread */
        pthread_t t_tid;        /* the thread id */
        int fileno;             /* The filenumber for this stream */
        int pos;                /* The position in the file. */
#ifdef __SOLARIS__
        off64_t offset;         /* The file's current offset */
#else
        off_t offset;
#endif /* __SOLARIS */
        char buf[NUMBYTES+1];   /* The thread's buffer. */
        int errorcnt;           /* The number of errors from this stream */
} streamdata[MAX_STREAMS];

/* The device file descriptor. */
int dev_fd;

/* Size of a file in bytes, From stat.  Assumes all files are the same size. */
int how_big;

/* A count of the number of errors in each round.  */
int round_errors[MAX_NUM_ROUNDS];

pthread_cond_t sched_go;
pthread_cond_t sched_alldone;
pthread_cond_t sched_round_done;
pthread_mutex_t sched_mutex;

/* The following variables are used for stream synchronization. */
int current_round = -1; 
/* The number of streams completed in the current round */
int num_completed = 0;          

pthread_t sched_tid;

void *readStream(void *fnp);
void *scheduler(void *idp);
int readMeg(char *buf, int streamno);

struct timeval start_tv;

int main(int argc, char *argv[]) {
        int i = 0;
	int rv, sum;

        pthread_attr_t attr;

        /* see options.c */
        extern int num_streams, num_blocks;

        /* see options.c */
        parseArgs(argc, argv);
        if(validateParams() != VALIDATION_SUCCESS) {
                printUsage();
                exit(1);
        }

	/* Initialization */
	how_big = num_blocks;

        /* RNG */
        srandom(1);

	/* Open the device */
#ifdef __SOLARIS__
        dev_fd = open64(RAWDEV, O_RDONLY | O_LARGEFILE, NULL);
#else
        dev_fd = open(RAWDEV, O_RDONLY, NULL);
#endif
        
        if(dev_fd < 1) { 
                printf("open error");
                exit(1);
        }

        /* Initialize the per round error counter. */
        for(i = 0; i < MAX_NUM_ROUNDS; i++) {
                round_errors[i] = 0;
        }

	rv = pthread_attr_init(&attr);
        if(rv) {
                perror("Can't init pthread attributes.");
                exit(1);
        }

	/* TODO: is this supported on Solaris8? */
	rv = pthread_attr_setschedpolicy(&attr, SCHED_RR); 
	// rv = pthread_attr_setschedpolicy(&attr, SCHED_FIFO); 
	if(rv) {
		perror("Can't set SCHED_RR policy");
	}

 	// rv = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
	if(rv) {
                perror("Can't set thread scope to system-wide scope.");
                exit(1);
        }
		
        pthread_cond_init(&sched_go, NULL);
        pthread_mutex_init(&sched_mutex, NULL);
        pthread_cond_init(&sched_round_done, NULL);
        pthread_cond_init(&sched_alldone, NULL);

        for (i = 0; i < num_streams; i++) {
                streamdata[i].fileno = i;
                /* Each stream starts at a random position within it's buffer area for a more realistic simulation */
                streamdata[i].pos = random() % how_big;
                /* Each stream's buffer area is spaced across the whole disk.
                 * The disks we're using have DISKSIZE MEGS, so we space across that. */
		streamdata[i].offset = ( (unsigned long long) i * ((DISKSIZE / 100) / num_streams)  ) * ONE_MEGA * (unsigned long long)how_big;
#ifdef DEBUGVERBOSE
                printf("Creating thread: %d with offset 0x%llX and position %d\n",i, streamdata[i].offset, streamdata[i].pos);
#endif
                rv = pthread_create(&(streamdata[i].t_tid), &attr, readStream, 
                                        (void *)&(streamdata[i].fileno));
                if(rv != 0) {
			switch(rv) {
			case NULL:
				printf("Not sane.\n");
				break;
			case ENOMEM:
				printf("Not enough memory.\n");
				break;
			case EINVAL:
				printf("Invalid attr.\n");
				break;
			case EPERM:
				printf("Invalid permissions.\n");
				break;
			default:
				printf("Other: %d.\n", rv);
			}
                        perror("Can't create thread");
                        exit(1);
                }
        }
	
        rv = pthread_create(&sched_tid, &attr, scheduler, NULL);
	if(rv != 0) {
		perror("Can't create scheduler thread");
		exit(1);
	}

        for(i = 0; i < num_streams; i++) {
#ifdef DEBUG
                printf("Waiting for thread: %d\n",i);
#endif
                pthread_join(streamdata[i].t_tid, NULL);
        }

	pthread_join(sched_tid, NULL);

        /* Process error information */
        sum = 0;
#ifdef DEBUG
        for (i = 0; i < how_big; i++) {
                printf("Errors in round %d: %d\n", i, round_errors[i]);
                sum += round_errors[i];
        }
        printf("Total round errors: %d\n", sum);
        sum = 0;
        for (i = 0; i < num_streams; i++) {
                printf("Errors in stream %d: %d\n", i, streamdata[i].errorcnt);
                sum += streamdata[i].errorcnt;
        }
        printf("Total round errors: %d\n", sum);
#endif
                
	exit(0);
}

void *readStream(void *fnp) {
        int i;
        char *buf;
	int ready = 0;

	// extern int num_streams;

        int fn = *(int *)fnp;

        buf = streamdata[fn].buf;

#ifdef DEBUGVERBOSE
        printf("Reading stream: %d\n", fn);
#endif

        memset(buf, '\0', NUMBYTES);

        streamdata[fn].errorcnt = 0;

        for (i = 0; i < how_big; i++) {
                pthread_mutex_lock(&sched_mutex);
		ready = (i <= current_round);
		pthread_mutex_unlock(&sched_mutex);
                while(!ready) {
			pthread_mutex_lock(&sched_mutex);
                        pthread_cond_wait(&sched_go,&sched_mutex);
			ready = (i <= current_round);
			pthread_mutex_unlock(&sched_mutex);
                }
#ifdef DEBUGVERBOSE
                printf("Reading chunk %d in stream %d.\n",i, fn);
#endif
                if(readMeg(buf, fn) < 0) {
                        fprintf(stderr, "Exiting with i = %d.\n", i);
                        exit(1);
                }
#ifdef DEBUGVERBOSE
                fprintf (stderr, "Stream %d block %d completed.\n", fn, i);
#endif
                // printf(".");
	
		pthread_mutex_lock(&sched_mutex);
                ++num_completed;
                pthread_cond_signal(&sched_round_done);
                pthread_mutex_unlock(&sched_mutex);
        }
	return(0);
}

void *scheduler(void *idp) {
        // int *my_id = idp;
        int i, errctr = 0, done = 0;
	// struct timespec done_time;
	// int tmp_ntime, tmp_time;

        extern int block_size, num_streams, bit_rate;

        init_timer(block_size, bit_rate);
        init_since();

        for(i = 0; i < how_big; i++) {
                now_since();
#ifdef DEBUGVERBOSE
		fprintf(stderr,"Starting loop.\n");
#endif
                pthread_mutex_lock(&sched_mutex);
#ifdef DEBUGVERBOSE
		fprintf(stderr,"Got lock.\n");
#endif
		++current_round; /* Should == i */
                num_completed = 0;
		pthread_cond_broadcast(&sched_go);
#ifdef DEBUGVERBOSE
		fprintf(stderr,"Sent broadcast.\n");
#endif
		pthread_mutex_unlock(&sched_mutex);
#ifdef DEBUGVERBOSE
		fprintf(stderr,"Unlocked mutex, about to sleep.\n");
#endif

                pthread_mutex_lock(&sched_mutex);
                done = 0;
                while(!done) {
                        pthread_cond_wait(&sched_round_done, &sched_mutex);
#ifdef DEBUGVERBOSE
                        fprintf(stderr, "Some stream's done!\n");
#endif
                        done = (num_completed >= num_streams);
                }
#ifdef DEBUGVERBOSE
                fprintf(stderr, "All streams done!\n");
#endif
                since();
                if(sleep_end_quantum_increment()) {
                        ++errctr;
                }
                printf("Period %d, %d errors\n", i, errctr);
                pthread_mutex_unlock(&sched_mutex);
        }
        end_since();
        printf("Total errors: %d\n", errctr);
        return(0);
}

int readMeg(char *buf, int streamno) {
        int curByte = 0;
        int rv;
#ifdef __SOLARIS__
        off64_t offset;
#else
        off_t offset;
#endif /* __SOLARIS__ */

	extern int block_size;

        offset = streamdata[streamno].offset + ONE_MEGA * streamdata[streamno].pos;
        while(1) {
#ifdef DEBUGVERBOSE
                fprintf(stderr, "Commencing read %d, %u, %d, %d, 0x%llX, 0x%llX\n", streamno, (unsigned int)&buf, block_size, curByte, offset, streamdata[streamno].offset);
#endif
                
#ifdef __SOLARIS__
                rv = pread64(dev_fd, buf + curByte, block_size - curByte, offset);
#else
                rv = pread(dev_fd, buf + curByte, block_size - curByte, offset);
#endif /* __SOLARIS__ */
                
#ifdef DEBUGVERBOSE
		fprintf(stderr, "Finished read, %d\n", streamno);
#endif
                curByte += rv;
                if(rv > 0) {
#ifdef DEBUGVERBOSE
                        fprintf(stderr, "Read %d bytes.\n",rv);
#endif
                        offset += (unsigned long long)rv;
                        if (curByte < block_size) {
                                continue;
                        } else {
                                if(++streamdata[streamno].pos >= 100) {
                                        streamdata[streamno].pos = 0;
                                }
                                
                                return curByte;
                        }
                } else if (rv == 0) {
                        fprintf(stderr, "Read Zero bytes.\n");
                        return rv;
                } else {
                        fprintf(stderr, "Read returned negative: %d.\n", rv);
                        return rv;
                }
        }
        return rv;
}

