/******************************************************************************
* FILE: mpi_mm.c
* DESCRIPTION:  
*   MPI Matrix Multiply - C Version
*   In this code, the master task distributes a matrix multiply
*   operation to numtasks-1 worker tasks.
* Original Source: https://computing.llnl.gov/tutorials/mpi/exercise.html#Exercise1
* How to Compile and Run @euclid:
	[FROM any 103ws??] scp * ucycsu02@euclid.cyi.ac.cy:~/epl372
	[FROM any 103ws??] ssh ucycsu02@euclid.cyi.ac.cy
	[ucycsu02@euclid]$ cd epl372
	[ucycsu02@euclid epl372]$ module load goolf/1.6.10
	[ucycsu02@euclid epl372]$ mpicc mpi_mm.c -o mpi_mm.out
	[ucycsu02@euclid epl372]$ sbatch mpirun.sh
	[ucycsu02@euclid epl372]$ squeue

#!/bin/bash
#SBATCH --job-name=MPI_MM
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=5
#SBATCH --time=00:05:00
#SBATCH --error=mpi_mm.error.out
#SBATCH --output=mpi_mm.output.out
module load goolf/1.6.10
mpirun -np 8 ./mpi_mm.out

**** ALWAYS Compare the results with the Serial Version to VERIFY *****
******************************************************************************/
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>

#define NRA 10                 /* number of rows in matrix A */
#define NCA 10                 /* number of columns in matrix A */
#define NCB 10                  /* number of columns in matrix B */
#define MASTER 0               /* taskid of first task */
#define FROM_MASTER 1          /* setting a message type */
#define FROM_WORKER 2          /* setting a message type */

int main (int argc, char *argv[])
{
int	numtasks,              /* number of tasks in partition */
	taskid,                /* a task identifier */
	numworkers,            /* number of worker tasks */
	source,                /* task id of message source */
	dest,                  /* task id of message destination */
	mtype,                 /* message type */
	rows,                  /* rows of matrix A sent to each worker */
	averow, extra, offset, /* used to determine rows sent to each worker */
	i, j, k, rc;           /* misc */
	int  len;
	char hostname[MPI_MAX_PROCESSOR_NAME];
double	a[NRA][NCA],           /* matrix A to be multiplied */
		b[NCA][NCB],           /* matrix B to be multiplied */
		c[NRA][NCB];           /* result matrix C */
MPI_Status status;

MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&taskid);
MPI_Comm_size(MPI_COMM_WORLD,&numtasks);  
MPI_Get_processor_name(hostname, &len);

if (numtasks < 2 ) {
  printf("Need at least two MPI tasks. Quitting...\n");
  MPI_Abort(MPI_COMM_WORLD, rc);
  exit(1);
  }
numworkers = numtasks-1;


/**************************** master task ************************************/
   if (taskid == MASTER)
   {
      printf("mpi_mm has started with %d tasks.\n",numtasks);
      printf("Initializing arrays...A[%d][%d]*B[%d][%d]=C[%d][%d]\n",NRA,NCA,NCA,NCB,NRA,NCB);
      for (i=0; i<NRA; i++)
         for (j=0; j<NCA; j++)
            a[i][j]= i+j;
      for (i=0; i<NCA; i++)
         for (j=0; j<NCB; j++)
            b[i][j]= i*j;

      /* Send matrix data to the worker tasks */
      averow = NRA/numworkers;
      extra = NRA%numworkers;
      offset = 0;
      mtype = FROM_MASTER;
      for (dest=1; dest<=numworkers; dest++)
      {
         rows = (dest <= extra) ? averow+1 : averow;   	
         printf("Sending %d rows to task %d offset=%d\n",rows,dest,offset);
         MPI_Send(&offset, 1, MPI_INT, dest, mtype, MPI_COMM_WORLD);
         MPI_Send(&rows, 1, MPI_INT, dest, mtype, MPI_COMM_WORLD);
         MPI_Send(&a[offset][0], rows*NCA, MPI_DOUBLE, dest, mtype,
                   MPI_COMM_WORLD);
		/*** We need to send all the Columns of B to all the workers*/
        // MPI_Send(&b, NCA*NCB, MPI_DOUBLE, dest, mtype, MPI_COMM_WORLD);
         offset = offset + rows;
      }
	  /*** Or you can Boadcast matrix B, Source = MASTER*/
	  printf("Broadcasting Matrix B\n");
	  MPI_Bcast (&b, NCA*NCB, MPI_DOUBLE,MASTER,MPI_COMM_WORLD);

      /* Receive results from worker tasks */
      mtype = FROM_WORKER;
      for (i=1; i<=numworkers; i++)
      {
         source = i;
         MPI_Recv(&offset, 1, MPI_INT, source, mtype, MPI_COMM_WORLD, &status);
         MPI_Recv(&rows, 1, MPI_INT, source, mtype, MPI_COMM_WORLD, &status);
         MPI_Recv(&c[offset][0], rows*NCB, MPI_DOUBLE, source, mtype, 
                  MPI_COMM_WORLD, &status);
         printf("Received results from task %d\n",source);
      }

      /* Print results */
      printf("******************************************************\n");
      printf("Result Matrix:\n");
      for (i=0; i<NRA; i++)
      {
         printf("\n"); 
         for (j=0; j<NCB; j++) 
            printf("%6.2f   ", c[i][j]);
      }
      printf("\n******************************************************\n");
      printf ("Done.\n");
   }


/**************************** worker task ************************************/
   if (taskid > MASTER)
   {
      mtype = FROM_MASTER;
      MPI_Recv(&offset, 1, MPI_INT, MASTER, mtype, MPI_COMM_WORLD, &status);
      MPI_Recv(&rows, 1, MPI_INT, MASTER, mtype, MPI_COMM_WORLD, &status);
      MPI_Recv(&a, rows*NCA, MPI_DOUBLE, MASTER, mtype, MPI_COMM_WORLD, &status);
      //MPI_Recv(&b, NCA*NCB, MPI_DOUBLE, MASTER, mtype, MPI_COMM_WORLD, &status);
	  
	  /* Or you can Boadcast matrix B. SAME LINE AS MASTER (Source) */
	  MPI_Bcast(&b, NCA*NCB, MPI_DOUBLE, MASTER, MPI_COMM_WORLD);
	  
	  printf("Task %d on %s is working on offset %d, rows %d:\n",taskid, hostname, offset,rows);
      for (k=0; k<NCB; k++)
         for (i=0; i<rows; i++)
         {
            c[i][k] = 0.0;
            for (j=0; j<NCA; j++)
               c[i][k] = c[i][k] + a[i][j] * b[j][k];
         }
      mtype = FROM_WORKER;
      MPI_Send(&offset, 1, MPI_INT, MASTER, mtype, MPI_COMM_WORLD);
      MPI_Send(&rows, 1, MPI_INT, MASTER, mtype, MPI_COMM_WORLD);
      MPI_Send(&c, rows*NCB, MPI_DOUBLE, MASTER, mtype, MPI_COMM_WORLD);
   }
   MPI_Finalize();
}


/*************************************************************************************
mpi_mm has started with 10 tasks.
Initializing arrays...A[10][10]*B[10][10]=C[10][10]
Sending 2 rows to task 1 offset=0
Sending 1 rows to task 2 offset=2
Task 1 on e01 is working on offset 0, rows 2:
Sending 1 rows to task 3 offset=3
Task 2 on e02 is working on offset 2, rows 1:
Sending 1 rows to task 4 offset=4
Task 3 on e02 is working on offset 3, rows 1:
Sending 1 rows to task 5 offset=5
Task 4 on e03 is working on offset 4, rows 1:
Sending 1 rows to task 6 offset=6
Task 5 on e03 is working on offset 5, rows 1:
Sending 1 rows to task 7 offset=7
Task 6 on e04 is working on offset 6, rows 1:
Sending 1 rows to task 8 offset=8
Task 7 on e04 is working on offset 7, rows 1:
Sending 1 rows to task 9 offset=9
Task 8 on e05 is working on offset 8, rows 1:
Received results from task 1
Received results from task 2
Received results from task 3
Received results from task 4
Received results from task 5
Received results from task 6
Received results from task 7
Received results from task 8
Task 9 on e05 is working on offset 9, rows 1:
Received results from task 9
******************************************************
Result Matrix:

  0.00   285.00   570.00   855.00   1140.00   1425.00   1710.00   1995.00   2280.00   2565.00
  0.00   330.00   660.00   990.00   1320.00   1650.00   1980.00   2310.00   2640.00   2970.00
  0.00   375.00   750.00   1125.00   1500.00   1875.00   2250.00   2625.00   3000.00   3375.00
  0.00   420.00   840.00   1260.00   1680.00   2100.00   2520.00   2940.00   3360.00   3780.00
  0.00   465.00   930.00   1395.00   1860.00   2325.00   2790.00   3255.00   3720.00   4185.00
  0.00   510.00   1020.00   1530.00   2040.00   2550.00   3060.00   3570.00   4080.00   4590.00
  0.00   555.00   1110.00   1665.00   2220.00   2775.00   3330.00   3885.00   4440.00   4995.00
  0.00   600.00   1200.00   1800.00   2400.00   3000.00   3600.00   4200.00   4800.00   5400.00
  0.00   645.00   1290.00   1935.00   2580.00   3225.00   3870.00   4515.00   5160.00   5805.00
  0.00   690.00   1380.00   2070.00   2760.00   3450.00   4140.00   4830.00   5520.00   6210.00
******************************************************
Done.
*************************************************************************************/