Robust/src/Runtime/multicoretask.c

   1 #ifdef TASK
   2 #include "runtime.h"
   3 #ifndef RAW
   4 #include "structdefs.h"
   5 #include "mem.h"
   6 #include "checkpoint.h"
   7 #include "Queue.h"
   8 #include "SimpleHash.h"
   9 #include "GenericHashtable.h"
  10 #include <sys/select.h>
  11 #include <sys/types.h>
  12 #include <sys/mman.h>
  13 #include <string.h>
  14 #include <signal.h>
  15 #include <assert.h>
  16 #include <errno.h>
  17 #endif
  18 #ifdef RAW
  19 #include <raw.h>
  20 #include <raw_compiler_defs.h>
  21 //#include <libints.h>
  22 #elif defined THREADSIMULATE
  23 // use POSIX message queue
  24 // for each core, its message queue named as
  25 // /msgqueue_corenum
  26 #include <mqueue.h>
  27 #include <sys/stat.h>
  28 #endif
  29 /*
  30 extern int injectfailures;
  31 extern float failurechance;
  32 */
  33 extern int debugtask;
  34 extern int instaccum;
  35
  36 #ifdef RAW
  37 #define TOTALCORE raw_get_num_tiles()
  38 #endif
  39
  40 #ifdef CONSCHECK
  41 #include "instrument.h"
  42 #endif
  43
  44 struct genhashtable * activetasks;
  45 struct genhashtable * failedtasks;
  46 struct taskparamdescriptor * currtpd;
  47 #ifndef RAW
  48 struct RuntimeHash * forward;
  49 struct RuntimeHash * reverse;
  50 #endif
  51
  52 int corestatus[NUMCORES]; // records status of each core
  53                           // 1: running tasks
  54                                                   // 0: stall
  55 int numsendobjs[NUMCORES]; // records how many objects a core has sent out
  56 int numreceiveobjs[NUMCORES]; // records how many objects a core has received
  57 #ifdef RAW
  58 struct RuntimeHash locktable;
  59 static struct RuntimeHash* locktbl = &locktable;
  60 void * curr_heapbase=0;
  61 void * curr_heaptop=0;
  62 int self_numsendobjs;
  63 int self_numreceiveobjs;
  64 int lockobj;
  65 int lockresult;
  66 bool lockflag;
  67 #ifndef INTERRUPT
  68 bool reside;
  69 #endif
  70 struct Queue objqueue;
  71 int msgdata[30];
  72 int msgtype;
  73 int msgdataindex;
  74 int msglength;
  75 void calCoords(int core_num, int* coordY, int* coordX);
  76 #elif defined THREADSIMULATE
  77 static struct RuntimeHash* locktbl;
  78 struct thread_data {
  79         int corenum;
  80         int argc;
  81         char** argv;
  82         int numsendobjs;
  83         int numreceiveobjs;
  84 };
  85 struct thread_data thread_data_array[NUMCORES];
  86 mqd_t mqd[NUMCORES];
  87 static pthread_key_t key;
  88 static pthread_rwlock_t rwlock_tbl;
  89 static pthread_rwlock_t rwlock_init;
  90
  91 void run(void * arg);
  92 #endif
  93
  94 bool transStallMsg(int targetcore);
  95 void transTerminateMsg(int targetcore);
  96 int receiveObject();
  97 bool getreadlock(void* ptr);
  98 void releasereadlock(void* ptr);
  99 #ifdef RAW
 100 bool getreadlock_I(void* ptr);
 101 void releasereadlock_I(void* ptr);
 102 #endif
 103 bool getwritelock(void* ptr);
 104 void releasewritelock(void* ptr);
 105
 106 #ifdef RAW
 107 void begin() {
 108 #else
 109 int main(int argc, char **argv) {
 110 #endif
 111 #ifdef RAW
 112         int i = 0;
 113         int argc = 1;
 114         char ** argv = NULL;
 115         bool sendStall = false;
 116         bool isfirst = true;
 117         bool tocontinue = false;
 118         struct QueueItem * objitem = NULL;
 119         struct transObjInfo * objInfo = NULL;
 120         int grount = 0;
 121         bool allStall = true;
 122         int sumsendobj = 0;
 123
 124 #ifdef RAWDEBUG
 125         raw_test_pass(0xee01);
 126 #endif
 127         corenum = raw_get_abs_pos_x() + 4 * raw_get_abs_pos_y();
 128
 129         // initialize the arrays
 130         if(STARTUPCORE == corenum) {
 131                 // startup core to initialize corestatus[]
 132                 for(i = 0; i < NUMCORES; ++i) {
 133                         corestatus[i] = 1;
 134                         numsendobjs[i] = 0; // assume all variables in RAW are local variables! MAY BE WRONG!!!
 135                         numreceiveobjs[i] = 0;
 136                 }
 137         }
 138         self_numsendobjs = 0;
 139         self_numreceiveobjs = 0;
 140         for(i = 0; i < 30; ++i) {
 141                 msgdata[i] = -1;
 142         }
 143         //msgdata = NULL;
 144         msgtype = -1;
 145         msgdataindex = 0;
 146         msglength = 30;
 147 #ifdef RAWDEBUG
 148         raw_test_pass(0xee02);
 149 #endif
 150
 151         // create the lock table, lockresult table and obj queue
 152         locktable.size = 20;
 153     locktable.bucket = (struct RuntimeNode **) RUNMALLOC_I(sizeof(struct RuntimeNode *)*20);
 154     /* Set allocation blocks*/
 155     locktable.listhead=NULL;
 156     locktable.listtail=NULL;
 157     /*Set data counts*/
 158     locktable.numelements = 0;
 159         lockobj = 0;
 160         lockresult = 0;
 161         lockflag = false;
 162 #ifndef INTERRUPT
 163         reside = false;
 164 #endif
 165         objqueue.head = NULL;
 166         objqueue.tail = NULL;
 167 #ifdef RAWDEBUG
 168         raw_test_pass(0xee03);
 169 #endif
 170
 171 #ifdef INTERRUPT
 172         if (corenum < NUMCORES) {
 173         // set up interrupts
 174         setup_ints();
 175         //setup_interrupts();
 176         //start_gdn_avail_ints(recvMsg);
 177         raw_user_interrupts_on();
 178 #ifdef RAWDEBUG
 179         raw_test_pass(0xee04);
 180 #endif
 181         }
 182 #endif
 183
 184 #elif defined THREADSIMULATE
 185         errno = 0;
 186         int tids[NUMCORES];
 187         int rc[NUMCORES];
 188         pthread_t threads[NUMCORES];
 189         int i = 0;
 190
 191         // initialize three arrays and msg queue array
 192         char * pathhead = "/msgqueue_";
 193         int targetlen = strlen(pathhead);
 194         for(i = 0; i < NUMCORES; ++i) {
 195                 corestatus[i] = 1;
 196                 numsendobjs[i] = 0;
 197                 numreceiveobjs[i] = 0;
 198
 199                 char corenumstr[3];
 200                 int sourcelen = 0;
 201                 if(i < 10) {
 202                         corenumstr[0] = i + '0';
 203                         corenumstr[1] = '\0';
 204                         sourcelen = 1;
 205                 } else if(i < 100) {
 206                         corenumstr[1] = i %10 + '0';
 207                         corenumstr[0] = (i / 10) + '0';
 208                         corenumstr[2] = '\0';
 209                         sourcelen = 2;
 210                 } else {
 211                         printf("Error: i >= 100\n");
 212                         fflush(stdout);
 213                         exit(-1);
 214                 }
 215                 char path[targetlen + sourcelen + 1];
 216                 strcpy(path, pathhead);
 217                 strncat(path, corenumstr, sourcelen);
 218                 int oflags = O_RDONLY|O_CREAT|O_NONBLOCK;
 219                 int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
 220                 mq_unlink(path);
 221                 mqd[i]= mq_open(path, oflags, omodes, NULL);
 222                 if(mqd[i] == -1) {
 223                         printf("[Main] mq_open %s fails: %d, error: %s\n", path, mqd[i], strerror(errno));
 224                         exit(-1);
 225                 } else {
 226                         printf("[Main] mq_open %s returns: %d\n", path, mqd[i]);
 227                 }
 228         }
 229
 230         // create the key
 231         pthread_key_create(&key, NULL);
 232
 233         // create the lock table and initialize its mutex
 234         locktbl = allocateRuntimeHash(20);
 235         int rc_locktbl = pthread_rwlock_init(&rwlock_tbl, NULL);
 236         printf("[Main] initialize the rwlock for lock table: %d error: \n", rc_locktbl, strerror(rc_locktbl));
 237
 238         for(i = 0; i < NUMCORES; ++i) {
 239                 thread_data_array[i].corenum = i;
 240                 thread_data_array[i].argc = argc;
 241                 thread_data_array[i].argv = argv;
 242                 thread_data_array[i].numsendobjs = 0;
 243                 thread_data_array[i].numreceiveobjs = 0;
 244                 printf("[main] creating thread %d\n", i);
 245                 rc[i] = pthread_create(&threads[i], NULL, run, (void *)&thread_data_array[i]);
 246         if (rc[i]){
 247                         printf("[main] ERROR; return code from pthread_create() is %d\n", rc[i]);
 248                         fflush(stdout);
 249                         exit(-1);
 250                 }
 251         }
 252
 253         //pthread_exit(NULL);
 254         while(true) {}
 255 }
 256
 257 void run(void* arg) {
 258         struct thread_data * my_tdata = (struct thread_data *)arg;
 259         pthread_setspecific(key, (void *)my_tdata->corenum);
 260         int argc = my_tdata->argc;
 261         char** argv = my_tdata->argv;
 262         printf("[run, %d] Thread %d runs: %x\n", my_tdata->corenum, my_tdata->corenum, (int)pthread_self());
 263         fflush(stdout);
 264
 265 #endif
 266
 267 #ifdef BOEHM_GC
 268   GC_init(); // Initialize the garbage collector
 269 #endif
 270 #ifdef CONSCHECK
 271   initializemmap();
 272 #endif
 273 #ifndef RAW
 274   processOptions();
 275 #endif
 276   initializeexithandler();
 277 #ifdef RAWDEBUG
 278   raw_test_pass(0xee05);
 279 #endif
 280   /* Create table for failed tasks */
 281 #ifdef RAW
 282   if(corenum > NUMCORES - 1) {
 283           failedtasks = NULL;
 284           activetasks = NULL;
 285           while(true) {
 286                   receiveObject();
 287           }
 288   } else {
 289 #ifdef RAWDEBUG
 290           raw_test_pass(0xee06);
 291 #endif
 292 #endif
 293   /*failedtasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 294                                    (int (*)(void *,void *)) &comparetpd);*/
 295           failedtasks = NULL;
 296 #ifdef RAWDEBUG
 297   raw_test_pass(0xee07);
 298 #endif
 299   /* Create queue of active tasks */
 300   activetasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 301                                    (int (*)(void *,void *)) &comparetpd);
 302 #ifdef RAWDEBUG
 303   raw_test_pass(0xee08);
 304 #endif
 305
 306   /* Process task information */
 307   processtasks();
 308 #ifdef RAWDEBUG
 309   raw_test_pass(0xee09);
 310 #endif
 311
 312   if(STARTUPCORE == corenum) {
 313   /* Create startup object */
 314   createstartupobject(argc, argv);
 315   }
 316 #ifdef RAWDEBUG
 317   raw_test_pass(0xee0a);
 318 #endif
 319
 320 #ifdef RAW
 321 #ifdef RAWDEBUG
 322   raw_test_pass(0xee0b);
 323 #endif
 324
 325   while(true) {
 326 /*#ifndef INTERRUPT
 327           while(receiveObject() != -1) {
 328           }
 329 #endif*/
 330
 331           // check if there are new active tasks can be executed
 332           executetasks();
 333
 334 #ifndef INTERRUPT
 335           while(receiveObject() != -1) {
 336           }
 337 #endif
 338
 339 #ifdef RAWDEBUG
 340           raw_test_pass(0xee0c);
 341 #endif
 342
 343           // check if there are some pending objects, if yes, enqueue them and executetasks again
 344           tocontinue = false;
 345 #ifdef RAWDEBUG
 346           raw_test_pass(0xee0d);
 347 #endif
 348           while(!isEmpty(&objqueue)) {
 349                   void * obj = NULL;
 350 #ifdef INTERRUPT
 351                   raw_user_interrupts_off();
 352 #endif
 353 #ifdef RAWDEBUG
 354                   raw_test_pass(0xeee1);
 355 #endif
 356                   sendStall = false;
 357                   tocontinue = true;
 358                   objitem = getTail(&objqueue);
 359                   //obj = objitem->objectptr;
 360                   objInfo = (struct transObjInfo *)objitem->objectptr;
 361                   obj = objInfo->objptr;
 362 #ifdef RAWDEBUG
 363                   raw_test_pass_reg((int)obj);
 364 #endif
 365                   // grab lock and flush the obj
 366                   getreadlock_I(obj);
 367                   while(!lockflag) {
 368                           receiveObject();
 369                   }
 370                   grount = lockresult;
 371 #ifdef RAWDEBUG
 372                   raw_test_pass_reg(grount);
 373 #endif
 374
 375                   lockresult = 0;
 376                   lockobj = 0;
 377                   lockflag = false;
 378 #ifndef INTERRUPT
 379                   reside = false;
 380 #endif
 381
 382                   if(grount == 1) {
 383                           int k = 0;
 384                           raw_invalidate_cache_range(obj, classsize[((struct ___Object___ *)obj)->type]);
 385                           // flush the obj
 386                           /*for(k = 0; k < classsize[((struct ___Object___ *)obj)->type]; ++k) {
 387                                   invalidateAddr(obj + k);
 388                           }*/
 389                           // enqueue the object
 390                           for(k = 0; k < objInfo->length; ++k) {
 391                                   int taskindex = objInfo->queues[2 * k];
 392                                   int paramindex = objInfo->queues[2 * k + 1];
 393                                   struct parameterwrapper ** queues = &(paramqueues[corenum][taskindex][paramindex]);
 394 #ifdef RAWDEBUG
 395                                   raw_test_pass_reg(taskindex);
 396                                   raw_test_pass_reg(paramindex);
 397 #endif
 398                                   enqueueObject_I(obj, queues, 1);
 399                           }
 400                           removeItem(&objqueue, objitem);
 401                           releasereadlock_I(obj);
 402                           RUNFREE(objInfo->queues);
 403                           RUNFREE(objInfo);
 404                           /*enqueueObject_I(obj, NULL, 0);
 405                           removeItem(&objqueue, objitem);
 406                           releasereadlock_I(obj);*/
 407                   } else {
 408                           // can not get lock
 409                           // put it at the end of the queue
 410                           // and try to execute active tasks already enqueued first
 411                           removeItem(&objqueue, objitem);
 412                           addNewItem_I(&objqueue, objInfo);
 413 #ifdef INTERRUPT
 414                           raw_user_interrupts_on();
 415 #endif
 416                           break;
 417                   }
 418 #ifdef INTERRUPT
 419                   raw_user_interrupts_on();
 420 #endif
 421 #ifdef RAWDEBUG
 422                   raw_test_pass(0xee0e);
 423 #endif
 424           }
 425 #ifdef RAWDEBUG
 426           raw_test_pass(0xee0f);
 427 #endif
 428
 429           if(!tocontinue) {
 430                   // check if stop
 431                   if(STARTUPCORE == corenum) {
 432                           if(isfirst) {
 433 #ifdef RAWDEBUG
 434                                   raw_test_pass(0xee10);
 435 #endif
 436                                   isfirst = false;
 437                           }
 438 #ifdef INTERRUPT
 439                           raw_user_interrupts_off();
 440 #endif
 441                           corestatus[corenum] = 0;
 442                           numsendobjs[corenum] = self_numsendobjs;
 443                           numreceiveobjs[corenum] = self_numreceiveobjs;
 444                           // check the status of all cores
 445                           allStall = true;
 446 #ifdef RAWDEBUG
 447                           raw_test_pass_reg(NUMCORES);
 448 #endif
 449                           for(i = 0; i < NUMCORES; ++i) {
 450 #ifdef RAWDEBUG
 451                                   raw_test_pass(0xe000 + corestatus[i]);
 452 #endif
 453                                   if(corestatus[i] != 0) {
 454                                           allStall = false;
 455                                           break;
 456                                   }
 457                           }
 458                           if(allStall) {
 459                                   // check if the sum of send objs and receive obj are the same
 460                                   // yes->terminate
 461                                   // no->go on executing
 462                                   sumsendobj = 0;
 463                                   for(i = 0; i < NUMCORES; ++i) {
 464                                           sumsendobj += numsendobjs[i];
 465 #ifdef RAWDEBUG
 466                                           raw_test_pass(0xf000 + numsendobjs[i]);
 467 #endif
 468                                   }
 469                                   for(i = 0; i < NUMCORES; ++i) {
 470                                           sumsendobj -= numreceiveobjs[i];
 471 #ifdef RAWDEBUG
 472                                           raw_test_pass(0xf000 + numreceiveobjs[i]);
 473 #endif
 474                                   }
 475                                   if(0 == sumsendobj) {
 476                                           // terminate
 477 #ifdef RAWDEBUG
 478                                           raw_test_pass(0xee11);
 479 #endif
 480                                           raw_test_pass(raw_get_cycle());
 481                                           raw_test_done(1);     // All done.
 482                                   }
 483                           }
 484 #ifdef INTERRUPT
 485                           raw_user_interrupts_on();
 486 #endif
 487                   } else {
 488                           if(!sendStall) {
 489 #ifdef RAWDEBUG
 490                                   raw_test_pass(0xee12);
 491 #endif
 492                                   if(isfirst) {
 493                                           // wait for some time
 494                                           int halt = 10000;
 495 #ifdef RAWDEBUG
 496                                           raw_test_pass(0xee13);
 497 #endif
 498                                           while(halt--){}
 499                                           isfirst = false;
 500 #ifdef RAWDEBUG
 501                                           raw_test_pass(0xee14);
 502 #endif
 503                                   } else {
 504                                         // send StallMsg to startup core
 505 #ifdef RAWDEBUG
 506                                         raw_test_pass(0xee15);
 507 #endif
 508                                         sendStall = transStallMsg(STARTUPCORE);
 509                                         isfirst = true;
 510                                   }
 511                           } else {
 512                                   isfirst = true;
 513 #ifdef RAWDEBUG
 514                                   raw_test_pass(0xee16);
 515 #endif
 516                           }
 517                   }
 518           }
 519   }
 520   }
 521 #elif defined THREADSIMULATE
 522   /* Start executing the tasks */
 523   executetasks();
 524
 525   int i = 0;
 526   // check if there are new objects coming
 527   bool sendStall = false;
 528
 529   int numofcore = pthread_getspecific(key);
 530   while(true) {
 531           switch(receiveObject()) {
 532                   case 0: {
 533                                           printf("[run, %d] receive an object\n", numofcore);
 534                                           sendStall = false;
 535                                           // received an object
 536                                           // check if there are new active tasks can be executed
 537                                           executetasks();
 538                                           break;
 539                                   }
 540                   case 1: {
 541                                           //printf("[run, %d] no msg\n", numofcore);
 542                                           // no msg received
 543                                           if(STARTUPCORE == numofcore) {
 544                                                   corestatus[numofcore] = 0;
 545                                                   // check the status of all cores
 546                                                   bool allStall = true;
 547                                                   for(i = 0; i < NUMCORES; ++i) {
 548                                                           if(corestatus[i] != 0) {
 549                                                                   allStall = false;
 550                                                                   break;
 551                                                           }
 552                                                   }
 553                                                   if(allStall) {
 554                                                           // check if the sum of send objs and receive obj are the same
 555                                                           // yes->terminate
 556                                                           // no->go on executing
 557                                                           int sumsendobj = 0;
 558                                                           for(i = 0; i < NUMCORES; ++i) {
 559                                                                   sumsendobj += numsendobjs[i];
 560                                                           }
 561                                                           for(i = 0; i < NUMCORES; ++i) {
 562                                                                   sumsendobj -= numreceiveobjs[i];
 563                                                           }
 564                                                           if(0 == sumsendobj) {
 565                                                                   // terminate
 566
 567                                                                   // release all locks
 568                                                                   int rc_tbl = pthread_rwlock_wrlock(&rwlock_tbl);
 569                                                                   printf("[run, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc_tbl, strerror(rc_tbl));
 570                                                                   struct RuntimeIterator* it_lock = RuntimeHashcreateiterator(locktbl);
 571                                                                   while(0 != RunhasNext(it_lock)) {
 572                                                                           int key = Runkey(it_lock);
 573                                                                           pthread_rwlock_t* rwlock_obj = (pthread_rwlock_t*)Runnext(it_lock);
 574                                                                           int rc_des = pthread_rwlock_destroy(rwlock_obj);
 575                                                                           printf("[run, %d] destroy the rwlock for object: %d error: \n", numofcore, key, strerror(rc_des));
 576                                                                           RUNFREE(rwlock_obj);
 577                                                                   }
 578                                                                   freeRuntimeHash(locktbl);
 579                                                                   locktbl = NULL;
 580                                                                   RUNFREE(it_lock);
 581
 582                                                                   // destroy all message queues
 583                                                                   char * pathhead = "/msgqueue_";
 584                                                                   int targetlen = strlen(pathhead);
 585                                                                   for(i = 0; i < NUMCORES; ++i) {
 586                                                                           char corenumstr[3];
 587                                                                           int sourcelen = 0;
 588                                                                           if(i < 10) {
 589                                                                                   corenumstr[0] = i + '0';
 590                                                                                   corenumstr[1] = '\0';
 591                                                                                   sourcelen = 1;
 592                                                                           } else if(i < 100) {
 593                                                                                   corenumstr[1] = i %10 + '0';
 594                                                                                   corenumstr[0] = (i / 10) + '0';
 595                                                                                   corenumstr[2] = '\0';
 596                                                                                   sourcelen = 2;
 597                                                                           } else {
 598                                                                                   printf("Error: i >= 100\n");
 599                                                                                   fflush(stdout);
 600                                                                                   exit(-1);
 601                                                                           }
 602                                                                           char path[targetlen + sourcelen + 1];
 603                                                                           strcpy(path, pathhead);
 604                                                                           strncat(path, corenumstr, sourcelen);
 605                                                                           mq_unlink(path);
 606                                                                   }
 607
 608                                                                   printf("[run, %d] terminate!\n", numofcore);
 609                                                                   fflush(stdout);
 610                                                                   exit(0);
 611                                                           }
 612                                                   }
 613                                           } else {
 614                                                   if(!sendStall) {
 615                                                           // send StallMsg to startup core
 616                                                           sendStall = transStallMsg(STARTUPCORE);
 617                                                   }
 618                                           }
 619                                           break;
 620                                   }
 621                   case 2: {
 622                                           printf("[run, %d] receive a stall msg\n", numofcore);
 623                                           // receive a Stall Msg, do nothing
 624                                           assert(STARTUPCORE == numofcore); // only startup core can receive such msg
 625                                           sendStall = false;
 626                                           break;
 627                                   }
 628                  /* case 3: {
 629                                           printf("[run, %d] receive a terminate msg\n", numofcore);
 630                                           // receive a terminate Msg
 631                                           assert(STARTUPCORE != corenum); // only non-startup core can receive such msg
 632                                           mq_close(mqd[corenum]);
 633                                           fflush(stdout);
 634                                           exit(0);
 635                                           break;
 636                                   }*/
 637                   default: {
 638                                            printf("[run, %d] Error: invalid message type.\n", numofcore);
 639                                            fflush(stdout);
 640                                            exit(-1);
 641                                            break;
 642                                    }
 643           }
 644   }
 645 #endif
 646 }
 647
 648 void createstartupobject(int argc, char ** argv) {
 649   int i;
 650
 651   /* Allocate startup object     */
 652 #ifdef PRECISE_GC
 653   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(NULL, STARTUPTYPE);
 654   struct ArrayObject * stringarray=allocate_newarray(NULL, STRINGARRAYTYPE, argc-1);
 655 #else
 656   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(STARTUPTYPE);
 657   struct ArrayObject * stringarray=allocate_newarray(STRINGARRAYTYPE, argc-1);
 658 #endif
 659   /* Build array of strings */
 660   startupobject->___parameters___=stringarray;
 661   for(i=1;i<argc;i++) {
 662     int length=strlen(argv[i]);
 663 #ifdef PRECISE_GC
 664     struct ___String___ *newstring=NewString(NULL, argv[i],length);
 665 #else
 666     struct ___String___ *newstring=NewString(argv[i],length);
 667 #endif
 668     ((void **)(((char *)& stringarray->___length___)+sizeof(int)))[i-1]=newstring;
 669   }
 670
 671   startupobject->isolate = 1;
 672   startupobject->version = 0;
 673
 674   /* Set initialized flag for startup object */
 675   flagorandinit(startupobject,1,0xFFFFFFFF);
 676   enqueueObject(startupobject, NULL, 0);
 677 #ifdef RAW
 678   //flushAll();
 679   raw_flush_entire_cache();
 680 #endif
 681 }
 682
 683 int hashCodetpd(struct taskparamdescriptor *ftd) {
 684   int hash=(int)ftd->task;
 685   int i;
 686   for(i=0;i<ftd->numParameters;i++){
 687     hash^=(int)ftd->parameterArray[i];
 688   }
 689   return hash;
 690 }
 691
 692 int comparetpd(struct taskparamdescriptor *ftd1, struct taskparamdescriptor *ftd2) {
 693   int i;
 694   if (ftd1->task!=ftd2->task)
 695     return 0;
 696   for(i=0;i<ftd1->numParameters;i++)
 697     if(ftd1->parameterArray[i]!=ftd2->parameterArray[i])
 698       return 0;
 699   return 1;
 700 }
 701
 702 /* This function sets a tag. */
 703 #ifdef PRECISE_GC
 704 void tagset(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 705 #else
 706 void tagset(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 707 #endif
 708         struct ArrayObject * ao=NULL;
 709   struct ___Object___ * tagptr=obj->___tags___;
 710 #ifdef RAWDEBUG
 711   raw_test_pass(0xebb0);
 712 #endif
 713   if (tagptr==NULL) {
 714 #ifdef RAWDEBUG
 715           raw_test_pass(0xebb1);
 716 #endif
 717     obj->___tags___=(struct ___Object___ *)tagd;
 718   } else {
 719     /* Have to check if it is already set */
 720     if (tagptr->type==TAGTYPE) {
 721       struct ___TagDescriptor___ * td=(struct ___TagDescriptor___ *) tagptr;
 722 #ifdef RAWDEBUG
 723           raw_test_pass(0xebb2);
 724 #endif
 725       if (td==tagd) {
 726 #ifdef RAWDEBUG
 727                   raw_test_pass(0xebb3);
 728 #endif
 729         return;
 730           }
 731 #ifdef PRECISE_GC
 732       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 733       struct ArrayObject * ao=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL);
 734       obj=(struct ___Object___ *)ptrarray[2];
 735       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 736       td=(struct ___TagDescriptor___ *) obj->___tags___;
 737 #else
 738 #ifdef RAWDEBUG
 739           raw_test_pass(0xebb4);
 740 #endif
 741       ao=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL);
 742 #endif
 743 #ifdef RAWDEBUG
 744           raw_test_pass(0xebb5);
 745 #endif
 746       ARRAYSET(ao, struct ___TagDescriptor___ *, 0, td);
 747       ARRAYSET(ao, struct ___TagDescriptor___ *, 1, tagd);
 748       obj->___tags___=(struct ___Object___ *) ao;
 749       ao->___cachedCode___=2;
 750 #ifdef RAWDEBUG
 751           raw_test_pass(0xebb6);
 752 #endif
 753     } else {
 754       /* Array Case */
 755       int i;
 756       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 757 #ifdef RAWDEBUG
 758           raw_test_pass(0xebb7);
 759 #endif
 760       for(i=0;i<ao->___cachedCode___;i++) {
 761         struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___*, i);
 762 #ifdef RAWDEBUG
 763                 raw_test_pass(0xebb8);
 764 #endif
 765         if (td==tagd) {
 766 #ifdef RAWDEBUG
 767                 raw_test_pass(0xebb9);
 768 #endif
 769           return;
 770         }
 771       }
 772       if (ao->___cachedCode___<ao->___length___) {
 773 #ifdef RAWDEBUG
 774                 raw_test_pass(0xebba);
 775 #endif
 776         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, tagd);
 777         ao->___cachedCode___++;
 778 #ifdef RAWDEBUG
 779         raw_test_pass(0xebbb);
 780 #endif
 781       } else {
 782 #ifdef PRECISE_GC
 783         int ptrarray[]={2,(int) ptr, (int) obj, (int) tagd};
 784         struct ArrayObject * aonew=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 785         obj=(struct ___Object___ *)ptrarray[2];
 786         tagd=(struct ___TagDescriptor___ *) ptrarray[3];
 787         ao=(struct ArrayObject *)obj->___tags___;
 788 #else
 789         struct ArrayObject * aonew=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 790 #endif
 791 #ifdef RAWDEBUG
 792                 raw_test_pass(0xebbc);
 793 #endif
 794         aonew->___cachedCode___=ao->___length___+1;
 795         for(i=0;i<ao->___length___;i++) {
 796 #ifdef RAWDEBUG
 797                 raw_test_pass(0xebbd);
 798 #endif
 799           ARRAYSET(aonew, struct ___TagDescriptor___*, i, ARRAYGET(ao, struct ___TagDescriptor___*, i));
 800         }
 801 #ifdef RAWDEBUG
 802                 raw_test_pass(0xebbe);
 803 #endif
 804         ARRAYSET(aonew, struct ___TagDescriptor___ *, ao->___length___, tagd);
 805 #ifdef RAWDEBUG
 806         raw_test_pass(0xebbf);
 807 #endif
 808       }
 809     }
 810   }
 811
 812   {
 813     struct ___Object___ * tagset=tagd->flagptr;
 814 #ifdef RAWDEBUG
 815         raw_test_pass(0xb008);
 816 #endif
 817     if(tagset==NULL) {
 818 #ifdef RAWDEBUG
 819                 raw_test_pass(0xb009);
 820 #endif
 821       tagd->flagptr=obj;
 822     } else if (tagset->type!=OBJECTARRAYTYPE) {
 823 #ifdef PRECISE_GC
 824       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 825       struct ArrayObject * ao=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 826       obj=(struct ___Object___ *)ptrarray[2];
 827       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 828 #else
 829       struct ArrayObject * ao=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 830 #endif
 831       ARRAYSET(ao, struct ___Object___ *, 0, tagd->flagptr);
 832       ARRAYSET(ao, struct ___Object___ *, 1, obj);
 833       ao->___cachedCode___=2;
 834       tagd->flagptr=(struct ___Object___ *)ao;
 835 #ifdef RAWDEBUG
 836           raw_test_pass(0xb00a);
 837 #endif
 838     } else {
 839       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 840       if (ao->___cachedCode___<ao->___length___) {
 841 #ifdef RAWDEBUG
 842                   raw_test_pass(0xb00b);
 843 #endif
 844         ARRAYSET(ao, struct ___Object___*, ao->___cachedCode___++, obj);
 845       } else {
 846         int i;
 847 #ifdef PRECISE_GC
 848         int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 849         struct ArrayObject * aonew=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL+ao->___length___);
 850         obj=(struct ___Object___ *)ptrarray[2];
 851         tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 852         ao=(struct ArrayObject *)tagd->flagptr;
 853 #else
 854         struct ArrayObject * aonew=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 855 #endif
 856         aonew->___cachedCode___=ao->___cachedCode___+1;
 857         for(i=0;i<ao->___length___;i++) {
 858           ARRAYSET(aonew, struct ___Object___*, i, ARRAYGET(ao, struct ___Object___*, i));
 859         }
 860         ARRAYSET(aonew, struct ___Object___ *, ao->___cachedCode___, obj);
 861         tagd->flagptr=(struct ___Object___ *) aonew;
 862 #ifdef RAWDEBUG
 863         raw_test_pass(0xb00c);
 864 #endif
 865       }
 866     }
 867   }
 868 }
 869
 870 /* This function clears a tag. */
 871 #ifdef PRECISE_GC
 872 void tagclear(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 873 #else
 874 void tagclear(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 875 #endif
 876   /* We'll assume that tag is alway there.
 877      Need to statically check for this of course. */
 878   struct ___Object___ * tagptr=obj->___tags___;
 879
 880   if (tagptr->type==TAGTYPE) {
 881     if ((struct ___TagDescriptor___ *)tagptr==tagd)
 882       obj->___tags___=NULL;
 883     else
 884 #ifndef RAW
 885       printf("ERROR 1 in tagclear\n");
 886 #endif
 887         ;
 888   } else {
 889     struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 890     int i;
 891     for(i=0;i<ao->___cachedCode___;i++) {
 892       struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___ *, i);
 893       if (td==tagd) {
 894         ao->___cachedCode___--;
 895         if (i<ao->___cachedCode___)
 896           ARRAYSET(ao, struct ___TagDescriptor___ *, i, ARRAYGET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___));
 897         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, NULL);
 898         if (ao->___cachedCode___==0)
 899           obj->___tags___=NULL;
 900         goto PROCESSCLEAR;
 901       }
 902     }
 903 #ifndef RAW
 904     printf("ERROR 2 in tagclear\n");
 905 #endif
 906         ;
 907   }
 908  PROCESSCLEAR:
 909   {
 910     struct ___Object___ *tagset=tagd->flagptr;
 911     if (tagset->type!=OBJECTARRAYTYPE) {
 912       if (tagset==obj)
 913         tagd->flagptr=NULL;
 914       else
 915 #ifndef RAW
 916         printf("ERROR 3 in tagclear\n");
 917 #endif
 918           ;
 919     } else {
 920       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 921       int i;
 922       for(i=0;i<ao->___cachedCode___;i++) {
 923         struct ___Object___ * tobj=ARRAYGET(ao, struct ___Object___ *, i);
 924         if (tobj==obj) {
 925           ao->___cachedCode___--;
 926           if (i<ao->___cachedCode___)
 927             ARRAYSET(ao, struct ___Object___ *, i, ARRAYGET(ao, struct ___Object___ *, ao->___cachedCode___));
 928           ARRAYSET(ao, struct ___Object___ *, ao->___cachedCode___, NULL);
 929           if (ao->___cachedCode___==0)
 930             tagd->flagptr=NULL;
 931           goto ENDCLEAR;
 932         }
 933       }
 934 #ifndef RAW
 935       printf("ERROR 4 in tagclear\n");
 936 #endif
 937     }
 938   }
 939  ENDCLEAR:
 940   return;
 941 }
 942
 943 /* This function allocates a new tag. */
 944 #ifdef PRECISE_GC
 945 struct ___TagDescriptor___ * allocate_tag(void *ptr, int index) {
 946   struct ___TagDescriptor___ * v=(struct ___TagDescriptor___ *) mygcmalloc((struct garbagelist *) ptr, classsize[TAGTYPE]);
 947 #else
 948 struct ___TagDescriptor___ * allocate_tag(int index) {
 949   struct ___TagDescriptor___ * v=FREEMALLOC(classsize[TAGTYPE]);
 950 #endif
 951   v->type=TAGTYPE;
 952   v->flag=index;
 953   return v;
 954 }
 955
 956
 957
 958 /* This function updates the flag for object ptr.  It or's the flag
 959    with the or mask and and's it with the andmask. */
 960
 961 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** queues, int length, bool isnew);
 962
 963  int flagcomp(const int *val1, const int *val2) {
 964    return (*val1)-(*val2);
 965  }
 966
 967 void flagorand(void * ptr, int ormask, int andmask, struct parameterwrapper ** queues, int length) {
 968     {
 969       int oldflag=((int *)ptr)[1];
 970       int flag=ormask|oldflag;
 971       flag&=andmask;
 972 #ifdef RAWDEBUG
 973           raw_test_pass_reg((int)ptr);
 974           raw_test_pass(0xaa000000 + oldflag);
 975           raw_test_pass(0xaa000000 + flag);
 976 #endif
 977           flagbody(ptr, flag, queues, length, false);
 978     }
 979 }
 980
 981 bool intflagorand(void * ptr, int ormask, int andmask) {
 982     {
 983       int oldflag=((int *)ptr)[1];
 984       int flag=ormask|oldflag;
 985       flag&=andmask;
 986       if (flag==oldflag) /* Don't do anything */
 987         return false;
 988       else {
 989                  flagbody(ptr, flag, NULL, 0, false);
 990                  return true;
 991           }
 992     }
 993 }
 994
 995 void flagorandinit(void * ptr, int ormask, int andmask) {
 996   int oldflag=((int *)ptr)[1];
 997   int flag=ormask|oldflag;
 998   flag&=andmask;
 999 #ifdef RAWDEBUG
1000           raw_test_pass(0xaa100000 + oldflag);
1001           raw_test_pass(0xaa100000 + flag);
1002 #endif
1003   flagbody(ptr,flag,NULL,0,true);
1004 }
1005
1006 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** vqueues, int vlength, bool isnew) {
1007   struct parameterwrapper * flagptr = NULL;
1008   int i = 0;
1009   struct parameterwrapper ** queues = vqueues;
1010   int length = vlength;
1011   int next;
1012   int UNUSED, UNUSED2;
1013   int * enterflags = NULL;
1014   if((!isnew) && (queues == NULL)) {
1015 #ifdef THREADSIMULATE
1016           int numofcore = pthread_getspecific(key);
1017           queues = objectqueues[numofcore][ptr->type];
1018           length = numqueues[numofcore][ptr->type];
1019 #else
1020 #ifdef RAW
1021           if(corenum < NUMCORES) {
1022 #endif
1023           queues = objectqueues[corenum][ptr->type];
1024           length = numqueues[corenum][ptr->type];
1025 #ifdef RAW
1026           } else {
1027                   return;
1028           }
1029 #endif
1030 #endif
1031   }
1032   ptr->flag=flag;
1033 #ifdef RAWDEBUG
1034   raw_test_pass(0xbb000000 + ptr->flag);
1035 #endif
1036
1037   /*Remove object from all queues */
1038   for(i = 0; i < length; ++i) {
1039           flagptr = queues[i];
1040     ObjectHashget(flagptr->objectset, (int) ptr, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
1041     ObjectHashremove(flagptr->objectset, (int)ptr);
1042     if (enterflags!=NULL)
1043       RUNFREE(enterflags);
1044   }
1045  }
1046
1047  void enqueueObject(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1048    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1049
1050   {
1051     struct QueueItem *tmpptr;
1052         struct parameterwrapper * parameter=NULL;
1053         int j;
1054         int i;
1055     struct parameterwrapper * prevptr=NULL;
1056     struct ___Object___ *tagptr=NULL;
1057         struct parameterwrapper ** queues = vqueues;
1058         int length = vlength;
1059 #ifdef RAW
1060         if(corenum > NUMCORES - 1) {
1061                 return;
1062         }
1063 #endif
1064         if(queues == NULL) {
1065 #ifdef THREADSIMULATE
1066                 int numofcore = pthread_getspecific(key);
1067                 queues = objectqueues[numofcore][ptr->type];
1068                 length = numqueues[numofcore][ptr->type];
1069 #else
1070                 queues = objectqueues[corenum][ptr->type];
1071                 length = numqueues[corenum][ptr->type];
1072 #endif
1073         }
1074     tagptr=ptr->___tags___;
1075
1076     /* Outer loop iterates through all parameter queues an object of
1077        this type could be in.  */
1078     for(j = 0; j < length; ++j) {
1079                 parameter = queues[j];
1080       /* Check tags */
1081       if (parameter->numbertags>0) {
1082         if (tagptr==NULL)
1083           goto nextloop;//that means the object has no tag but that param needs tag
1084         else if(tagptr->type==TAGTYPE) {//one tag
1085           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1086           for(i=0;i<parameter->numbertags;i++) {
1087             //slotid is parameter->tagarray[2*i];
1088             int tagid=parameter->tagarray[2*i+1];
1089             if (tagid!=tagptr->flag)
1090               goto nextloop; /*We don't have this tag */
1091            }
1092         } else {//multiple tags
1093           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1094           for(i=0;i<parameter->numbertags;i++) {
1095             //slotid is parameter->tagarray[2*i];
1096             int tagid=parameter->tagarray[2*i+1];
1097             int j;
1098             for(j=0;j<ao->___cachedCode___;j++) {
1099               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag)
1100                 goto foundtag;
1101             }
1102             goto nextloop;
1103           foundtag:
1104             ;
1105           }
1106         }
1107       }
1108
1109       /* Check flags */
1110       for(i=0;i<parameter->numberofterms;i++) {
1111         int andmask=parameter->intarray[i*2];
1112         int checkmask=parameter->intarray[i*2+1];
1113         if ((ptr->flag&andmask)==checkmask) {
1114 #ifdef RAWDEBUG
1115                 raw_test_pass(0xcc000000 + andmask);
1116                 raw_test_pass_reg((int)ptr);
1117                 raw_test_pass(0xcc000000 + ptr->flag);
1118                 raw_test_pass(0xcc000000 + checkmask);
1119 #endif
1120           enqueuetasks(parameter, prevptr, ptr, NULL, 0);
1121           prevptr=parameter;
1122           break;
1123         }
1124       }
1125     nextloop:
1126           ;
1127     }
1128   }
1129 }
1130
1131 #ifdef RAW
1132 void enqueueObject_I(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1133    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1134
1135   {
1136     struct QueueItem *tmpptr;
1137         struct parameterwrapper * parameter=NULL;
1138         int j;
1139         int i;
1140     struct parameterwrapper * prevptr=NULL;
1141     struct ___Object___ *tagptr=NULL;
1142         struct parameterwrapper ** queues = vqueues;
1143         int length = vlength;
1144 #ifdef RAW
1145         if(corenum > NUMCORES - 1) {
1146                 return;
1147         }
1148 #endif
1149         if(queues == NULL) {
1150 #ifdef THREADSIMULATE
1151                 int numofcore = pthread_getspecific(key);
1152                 queues = objectqueues[numofcore][ptr->type];
1153                 length = numqueues[numofcore][ptr->type];
1154 #else
1155                 queues = objectqueues[corenum][ptr->type];
1156                 length = numqueues[corenum][ptr->type];
1157 #endif
1158         }
1159 #ifdef RAWDEBUG
1160         raw_test_pass(0xeaa1);
1161         raw_test_pass_reg(queues);
1162         raw_test_pass_reg(length);
1163 #endif
1164     tagptr=ptr->___tags___;
1165
1166     /* Outer loop iterates through all parameter queues an object of
1167        this type could be in.  */
1168     for(j = 0; j < length; ++j) {
1169                 parameter = queues[j];
1170       /* Check tags */
1171       if (parameter->numbertags>0) {
1172 #ifdef RAWDEBUG
1173         raw_test_pass(0xeaa2);
1174         raw_test_pass_reg(tagptr);
1175 #endif
1176         if (tagptr==NULL)
1177           goto nextloop;//that means the object has no tag but that param needs tag
1178         else if(tagptr->type==TAGTYPE) {//one tag
1179           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1180 #ifdef RAWDEBUG
1181         raw_test_pass(0xeaa3);
1182 #endif
1183           for(i=0;i<parameter->numbertags;i++) {
1184             //slotid is parameter->tagarray[2*i];
1185             int tagid=parameter->tagarray[2*i+1];
1186             if (tagid!=tagptr->flag) {
1187 #ifdef RAWDEBUG
1188         raw_test_pass(0xeaa4);
1189 #endif
1190               goto nextloop; /*We don't have this tag */
1191                 }
1192            }
1193         } else {//multiple tags
1194           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1195 #ifdef RAWDEBUG
1196         raw_test_pass(0xeaa5);
1197 #endif
1198           for(i=0;i<parameter->numbertags;i++) {
1199             //slotid is parameter->tagarray[2*i];
1200             int tagid=parameter->tagarray[2*i+1];
1201             int j;
1202             for(j=0;j<ao->___cachedCode___;j++) {
1203               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag) {
1204                 goto foundtag;
1205                   }
1206             }
1207 #ifdef RAWDEBUG
1208         raw_test_pass(0xeaa6);
1209 #endif
1210             goto nextloop;
1211           foundtag:
1212             ;
1213           }
1214         }
1215       }
1216
1217       /* Check flags */
1218       for(i=0;i<parameter->numberofterms;i++) {
1219         int andmask=parameter->intarray[i*2];
1220         int checkmask=parameter->intarray[i*2+1];
1221 #ifdef RAWDEBUG
1222         raw_test_pass(0xeaa7);
1223         raw_test_pass(0xcc000000 + andmask);
1224         raw_test_pass_reg(ptr);
1225         raw_test_pass(0xcc000000 + ptr->flag);
1226         raw_test_pass(0xcc000000 + checkmask);
1227 #endif
1228         if ((ptr->flag&andmask)==checkmask) {
1229 #ifdef RAWDEBUG
1230         raw_test_pass(0xeaa8);
1231 #endif
1232           enqueuetasks_I(parameter, prevptr, ptr, NULL, 0);
1233           prevptr=parameter;
1234           break;
1235         }
1236       }
1237     nextloop:
1238           ;
1239     }
1240   }
1241 }
1242
1243 // helper function to compute the coordinates of a core from the core number
1244 void calCoords(int core_num, int* coordY, int* coordX) {
1245         *coordX = core_num % 4;
1246         *coordY = core_num / 4;
1247 }
1248 #endif
1249
1250 /* Message format for RAW version:
1251  *      type + Msgbody
1252  * type: 0 -- transfer object
1253  *       1 -- transfer stall msg
1254  *       2 -- lock request
1255  *       3 -- lock grount
1256  *       4 -- lock deny
1257  *       5 -- lock release
1258  *
1259  * ObjMsg: 0 + size of msg + obj's address + (task index + param index)+
1260  * StallMsg: 1 + corenum + sendobjs + receiveobjs (size is always 4 * sizeof(int))
1261  * LockMsg: 2 + lock type + obj pointer + request core (size is always 4 * sizeof(int))
1262  *          3/4/5 + lock type + obj pointer (size is always 3 * sizeof(int))
1263  *          lock type: 0 -- read; 1 -- write
1264  */
1265
1266 // transfer an object to targetcore
1267 // format: object
1268 void transferObject(struct transObjInfo * transObj) {
1269         void * obj = transObj->objptr;
1270         int type=((int *)obj)[0];
1271     int size=classsize[type];
1272         int targetcore = transObj->targetcore;
1273         //assert(type < NUMCLASSES); // can only transfer normal object
1274
1275 #ifdef RAW
1276         unsigned msgHdr;
1277         int self_y, self_x, target_y, target_x;
1278         //int isshared = 0;
1279         // for 32 bit machine, the size of fixed part is always 3 words
1280         //int msgsize = sizeof(int) * 2 + sizeof(void *);
1281         int msgsize = 3 + transObj->length * 2;
1282         int i = 0;
1283
1284         struct ___Object___ * newobj = (struct ___Object___ *)obj;
1285         /*if(0 == newobj->isolate) {
1286                 isshared = 1;
1287         }*/
1288
1289         calCoords(corenum, &self_y, &self_x);
1290         calCoords(targetcore, &target_y, &target_x);
1291         // Build the message header
1292     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1293                                            self_y, self_x,
1294                                                            target_y, target_x);
1295         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1296 #ifdef RAWDEBUG
1297         raw_test_pass(0xbbbb);
1298         raw_test_pass(0xb000 + targetcore); // targetcore
1299 #endif
1300     gdn_send(0);
1301 #ifdef RAWDEBUG
1302     raw_test_pass(0);
1303 #endif
1304         gdn_send(msgsize);
1305 #ifdef RAWDEBUG
1306         raw_test_pass_reg(msgsize);
1307 #endif
1308     gdn_send(obj);
1309 #ifdef RAWDEBUG
1310         raw_test_pass_reg(obj);
1311 #endif
1312         for(i = 0; i < transObj->length; ++i) {
1313                 int taskindex = transObj->queues[2*i];
1314                 int paramindex = transObj->queues[2*i+1];
1315                 gdn_send(taskindex);
1316 #ifdef RAWDEBUG
1317                 raw_test_pass_reg(taskindex);
1318 #endif
1319                 gdn_send(paramindex);
1320 #ifdef RAWDEBUG
1321                 raw_test_pass_reg(paramindex);
1322 #endif
1323         }
1324 #ifdef RAWDEBUG
1325         raw_test_pass(0xffff);
1326 #endif
1327         ++(self_numsendobjs);
1328 #elif defined THREADSIMULATE
1329         int numofcore = pthread_getspecific(key);
1330
1331         // use POSIX message queue to transfer objects between cores
1332         mqd_t mqdnum;
1333         char corenumstr[3];
1334         int sourcelen = 0;
1335         if(targetcore < 10) {
1336                 corenumstr[0] = targetcore + '0';
1337                 corenumstr[1] = '\0';
1338                 sourcelen = 1;
1339         } else if(targetcore < 100) {
1340                 corenumstr[1] = targetcore % 10 + '0';
1341                 corenumstr[0] = (targetcore / 10) + '0';
1342                 corenumstr[2] = '\0';
1343                 sourcelen = 2;
1344         } else {
1345                 printf("Error: targetcore >= 100\n");
1346                 fflush(stdout);
1347                 exit(-1);
1348         }
1349         char * pathhead = "/msgqueue_";
1350         int targetlen = strlen(pathhead);
1351         char path[targetlen + sourcelen + 1];
1352         strcpy(path, pathhead);
1353         strncat(path, corenumstr, sourcelen);
1354         int oflags = O_WRONLY|O_NONBLOCK;
1355         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1356         mqdnum = mq_open(path, oflags, omodes, NULL);
1357         if(mqdnum==-1) {
1358                 printf("[transferObject, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1359                 fflush(stdout);
1360                 exit(-1);
1361         }
1362         /*struct ___Object___ * newobj = (struct ___Object___ *)obj;
1363         if(0 == newobj->isolate) {
1364                 newobj = RUNMALLOC(size);
1365                 memcpy(newobj, obj, size);
1366                 newobj->original=obj;
1367         }*/
1368         struct transObjInfo * tmptransObj = RUNMALLOC(sizeof(struct transObjInfo));
1369         memcpy(tmptransObj, transObj, sizeof(struct transObjInfo));
1370         int * tmpqueue = RUNMALLOC(sizeof(int)*2*tmptransObj->length);
1371         memcpy(tmpqueue, tmptransObj->queues, sizeof(int)*2*tmptransObj->length);
1372         tmptransObj->queues = tmpqueue;
1373         struct ___Object___ * newobj = RUNMALLOC(sizeof(struct ___Object___));
1374         newobj->type = ((struct ___Object___ *)obj)->type;
1375         newobj->original = (struct ___Object___ *)tmptransObj;
1376         int ret;
1377         do {
1378                 ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1379                 if(ret != 0) {
1380                         printf("[transferObject, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1381                 }
1382         }while(ret!=0);
1383         RUNFREE(newobj);
1384         if(numofcore == STARTUPCORE) {
1385                 ++numsendobjs[numofcore];
1386         } else {
1387                 ++(thread_data_array[numofcore].numsendobjs);
1388         }
1389         printf("[transferObject, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1390 #endif
1391 }
1392
1393 // send terminate message to targetcore
1394 // format: -1
1395 bool transStallMsg(int targetcore) {
1396 #ifdef RAW
1397         unsigned msgHdr;
1398         int self_y, self_x, target_y, target_x;
1399         // for 32 bit machine, the size is always 4 words
1400         //int msgsize = sizeof(int) * 4;
1401         int msgsize = 4;
1402
1403         calCoords(corenum, &self_y, &self_x);
1404         calCoords(targetcore, &target_y, &target_x);
1405         // Build the message header
1406     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1407                                            self_y, self_x,
1408                                                            target_y, target_x);
1409         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1410 #ifdef RAWDEBUG
1411         raw_test_pass(0xbbbb);
1412         raw_test_pass(0xb000 + targetcore); // targetcore
1413 #endif
1414     gdn_send(1);
1415 #ifdef RAWDEBUG
1416     raw_test_pass(1);
1417 #endif
1418     gdn_send(corenum);
1419 #ifdef RAWDEBUG
1420         raw_test_pass_reg(corenum);
1421 #endif
1422         gdn_send(self_numsendobjs);
1423 #ifdef RAWDEBUG
1424     raw_test_pass_reg(self_numsendobjs);
1425 #endif
1426     gdn_send(self_numreceiveobjs);
1427 #ifdef RAWDEBUG
1428         raw_test_pass_reg(self_numreceiveobjs);
1429         raw_test_pass(0xffff);
1430 #endif
1431         return true;
1432 #elif defined THREADSIMULATE
1433         struct ___Object___ *newobj = RUNMALLOC(sizeof(struct ___Object___));
1434         // use the first four int field to hold msgtype/corenum/sendobj/receiveobj
1435         newobj->type = -1;
1436         int numofcore = pthread_getspecific(key);
1437         newobj->flag = numofcore;
1438         newobj->___cachedHash___ = thread_data_array[numofcore].numsendobjs;
1439         newobj->___cachedCode___ = thread_data_array[numofcore].numreceiveobjs;
1440
1441         // use POSIX message queue to send stall msg to startup core
1442         assert(targetcore == STARTUPCORE);
1443         mqd_t mqdnum;
1444         char corenumstr[3];
1445         int sourcelen = 0;
1446         if(targetcore < 10) {
1447                 corenumstr[0] = targetcore + '0';
1448                 corenumstr[1] = '\0';
1449                 sourcelen = 1;
1450         } else if(targetcore < 100) {
1451                 corenumstr[1] = targetcore % 10 + '0';
1452                 corenumstr[0] = (targetcore / 10) + '0';
1453                 corenumstr[2] = '\0';
1454                 sourcelen = 2;
1455         } else {
1456                 printf("Error: targetcore >= 100\n");
1457                 fflush(stdout);
1458                 exit(-1);
1459         }
1460         char * pathhead = "/msgqueue_";
1461         int targetlen = strlen(pathhead);
1462         char path[targetlen + sourcelen + 1];
1463         strcpy(path, pathhead);
1464         strncat(path, corenumstr, sourcelen);
1465         int oflags = O_WRONLY|O_NONBLOCK;
1466         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1467         mqdnum = mq_open(path, oflags, omodes, NULL);
1468         if(mqdnum==-1) {
1469                 printf("[transStallMsg, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1470                 fflush(stdout);
1471                 exit(-1);
1472         }
1473         int ret;
1474         ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1475         if(ret != 0) {
1476                 printf("[transStallMsg, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1477                 RUNFREE(newobj);
1478                 return false;
1479         } else {
1480                 printf("[transStallMsg, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1481                 printf("<transStallMsg> to %s index: %d, sendobjs: %d, receiveobjs: %d\n", path, newobj->flag, newobj->___cachedHash___, newobj->___cachedCode___);
1482                 RUNFREE(newobj);
1483                 return true;
1484         }
1485 #endif
1486 }
1487
1488 // receive object transferred from other cores
1489 // or the terminate message from other cores
1490 // NOTICE: following format is for threadsimulate version only
1491 //         RAW version please see previous description
1492 // format: type + object
1493 // type: -1--stall msg
1494 //      !-1--object
1495 // return value: 0--received an object
1496 //               1--received nothing
1497 //               2--received a Stall Msg
1498 //               3--received a lock Msg
1499 //               RAW version: -1 -- received nothing
1500 //                            otherwise -- received msg type
1501 int receiveObject() {
1502 #ifdef RAW
1503         bool deny = false;
1504         unsigned msgHdr;
1505         int self_y, self_x, target_y, target_x;
1506         int targetcore = 0;
1507         if(gdn_input_avail() == 0) {
1508 #ifdef RAWDEBUG
1509                 if(corenum < NUMCORES) {
1510                         raw_test_pass(0xd001);
1511                 }
1512 #endif
1513                 return -1;
1514         }
1515 msg:
1516 #ifdef RAWDEBUG
1517         raw_test_pass(0xcccc);
1518 #endif
1519         while((gdn_input_avail() != 0) && (msgdataindex < msglength)) {
1520                 msgdata[msgdataindex] = gdn_receive();
1521                 if(msgdataindex == 0) {
1522                         if(msgdata[0] > 2) {
1523                                 msglength = 3;
1524                         } else if(msgdata[0] > 0) {
1525                                 msglength = 4;
1526                         }
1527                 } else if((msgdataindex == 1) && (msgdata[0] == 0)) {
1528                         msglength = msgdata[msgdataindex];
1529                 }
1530 #ifdef RAWDEBUG
1531                 raw_test_pass_reg(msgdata[msgdataindex]);
1532 #endif
1533                 msgdataindex++;
1534
1535                 /*if(msgdataindex == 0) {
1536                         // type
1537                         msgtype = gdn_receive();
1538                         if(msgtype > 2) {
1539                                 msglength = 3;
1540                         } else {
1541                                 msglength = 4;
1542                         }
1543                         if(msgtype != 0) {
1544                                 msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1545                                 msgdata[msgdataindex] = msgtype;
1546                         }
1547 #ifdef RAWDEBUG
1548                         raw_test_pass_reg(msgtype);
1549 #endif
1550                 } else if((msgdataindex == 1) && (msgtype == 0)) {
1551                         // object transfer msg
1552                         msglength = gdn_receive();
1553                         msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1554                         msgdata[0] = msgtype;
1555                         msgdata[msgdataindex] = msglength;
1556 #ifdef RAWDEBUG
1557                         raw_test_pass_reg(msgdata[msgdataindex]);
1558 #endif
1559                 } else {
1560                         msgdata[msgdataindex] = gdn_receive();
1561 #ifdef RAWDEBUG
1562                         raw_test_pass_reg(msgdata[msgdataindex]);
1563 #endif
1564                 }
1565                 msgdataindex++;*/
1566         }
1567 #ifdef RAWDEBUG
1568         raw_test_pass(0xffff);
1569 #endif
1570         if(msgdataindex == msglength) {
1571                 // received a whole msg
1572                 int type, data1, data2; // will receive at least 3 words including type
1573                 type = msgdata[0];
1574                 data1 = msgdata[1];
1575                 data2 = msgdata[2];
1576                 switch(type) {
1577                         case 0: {
1578                                                 // receive a object transfer msg
1579                                                 struct transObjInfo * transObj = RUNMALLOC_I(sizeof(struct transObjInfo));
1580                                                 int k = 0;
1581                                                 if(corenum > NUMCORES - 1) {
1582                                                         raw_test_done(0xa00a);
1583                                                 }
1584                                                 // store the object and its corresponding queue info, enqueue it later
1585                                                 transObj->objptr = (void *)data2; // data1 is now size of the msg
1586                                                 transObj->length = (msglength - 3) / 2;
1587                                                 transObj->queues = RUNMALLOC_I(sizeof(int)*(msglength - 3));
1588                                                 for(k = 0; k < transObj->length; ++k) {
1589                                                         transObj->queues[2*k] = msgdata[3+2*k];
1590 #ifdef RAWDEBUG
1591                                                         raw_test_pass_reg(transObj->queues[2*k]);
1592 #endif
1593                                                         transObj->queues[2*k+1] = msgdata[3+2*k+1];
1594 #ifdef RAWDEBUG
1595                                                         raw_test_pass_reg(transObj->queues[2*k+1]);
1596 #endif
1597                                                 }
1598                                                 //memcpy(transObj->queues, msgdata[3], sizeof(int)*(msglength - 3));
1599                                                 addNewItem_I(&objqueue, (void *)transObj);
1600                                                 ++(self_numreceiveobjs);
1601 #ifdef RAWDEBUG
1602                                                 raw_test_pass(0xe881);
1603 #endif
1604                                                 /*
1605                                                 addNewItem_I(&objqueue, (void *)data2);
1606                                                 ++(self_numreceiveobjs);
1607 #ifdef RAWDEBUG
1608                                                 raw_test_pass(0xe881);
1609 #endif
1610                         */
1611                                                 break;
1612                                         }
1613                         case 1: {
1614                                                 // receive a stall msg
1615                                                 if(corenum != STARTUPCORE) {
1616                                                         // non startup core can not receive stall msg
1617                                                         // return -1
1618                                                         raw_test_done(0xa001);
1619                                                 }
1620                                                 if(data1 < NUMCORES) {
1621 #ifdef RAWDEBUG
1622                                                         raw_test_pass(0xe882);
1623 #endif
1624                                                         corestatus[data1] = 0;
1625                                                         numsendobjs[data1] = data2;
1626                                                         numreceiveobjs[data1] = msgdata[3];
1627                                                 }
1628                                                 break;
1629                                         }
1630                         case 2: {
1631                                                 // receive lock request msg
1632                                                 // for 32 bit machine, the size is always 3 words
1633                                                 //int msgsize = sizeof(int) * 3;
1634                                                 int msgsize = 3;
1635                                                 // lock request msg, handle it right now
1636                                                 // check to see if there is a lock exist in locktbl for the required obj
1637                                                 int data3 = msgdata[3];
1638                                                 deny = false;
1639                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1640                                                         // no locks for this object
1641                                                         // first time to operate on this shared object
1642                                                         // create a lock for it
1643                                                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1644 #ifdef RAWDEBUG
1645                                                         raw_test_pass(0xe883);
1646 #endif
1647                                                         if(data1 == 0) {
1648                                                                 RuntimeHashadd_I(locktbl, data2, 1);
1649                                                         } else {
1650                                                                 RuntimeHashadd_I(locktbl, data2, -1);
1651                                                         }
1652                                                 } else {
1653                                                         int rwlock_obj = 0;
1654 #ifdef RAWDEBUG
1655                                                         raw_test_pass(0xe884);
1656 #endif
1657                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1658 #ifdef RAWDEBUG
1659                                                         raw_test_pass_reg(rwlock_obj);
1660 #endif
1661                                                         if(0 == rwlock_obj) {
1662                                                                 if(data1 == 0) {
1663                                                                         rwlock_obj = 1;
1664                                                                 } else {
1665                                                                         rwlock_obj = -1;
1666                                                                 }
1667                                                                 RuntimeHashremovekey(locktbl, data2);
1668                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1669                                                         } else if((rwlock_obj > 0) && (data1 == 0)) {
1670                                                                 // read lock request and there are only read locks
1671                                                                 rwlock_obj++;
1672                                                                 RuntimeHashremovekey(locktbl, data2);
1673                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1674                                                         } else {
1675                                                                 deny = true;
1676                                                         }
1677 #ifdef RAWDEBUG
1678                                                         raw_test_pass_reg(rwlock_obj);
1679 #endif
1680                                                 }
1681                                                 targetcore = data3;
1682                                                 calCoords(corenum, &self_y, &self_x);
1683                                                 calCoords(targetcore, &target_y, &target_x);
1684                                                 // Build the message header
1685                                                 msgHdr = construct_dyn_hdr(0, msgsize, 0,               // msgsize word sent.
1686                                                                                    self_y, self_x,
1687                                                                                                    target_y, target_x);
1688                                                 gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1689 #ifdef RAWDEBUG
1690                                                 raw_test_pass(0xbbbb);
1691                                                 raw_test_pass(0xb000 + targetcore); // targetcore
1692 #endif
1693                                                 if(deny == true) {
1694                                                         // deny the lock request
1695                                                         gdn_send(4); // lock request
1696 #ifdef RAWDEBUG
1697                                                         raw_test_pass(4);
1698 #endif
1699                                                 } else {
1700                                                         // grount the lock request
1701                                                         gdn_send(3); // lock request
1702 #ifdef RAWDEBUG
1703                                                         raw_test_pass(3);
1704 #endif
1705                                                 }
1706                                                 gdn_send(data1); // lock type
1707 #ifdef RAWDEBUG
1708                                                 raw_test_pass_reg(data1);
1709 #endif
1710                                                 gdn_send(data2); // lock target
1711 #ifdef RAWDEBUG
1712                                                 raw_test_pass_reg(data2);
1713                                                 raw_test_pass(0xffff);
1714 #endif
1715                                                 break;
1716                                         }
1717                         case 3: {
1718                                                 // receive lock grount msg
1719                                                 if(corenum > NUMCORES - 1) {
1720                                                         raw_test_done(0xa00b);
1721                                                 }
1722                                                 if(lockobj == data2) {
1723                                                         lockresult = 1;
1724                                                         lockflag = true;
1725 #ifndef INTERRUPT
1726                                                         reside = false;
1727 #endif
1728                                                 } else {
1729                                                         // conflicts on lockresults
1730                                                 raw_test_done(0xa002);
1731                                                 }
1732                                                 break;
1733                                         }
1734                         case 4: {
1735                                                 // receive lock grount/deny msg
1736                                                 if(corenum > NUMCORES - 1) {
1737                                                         raw_test_done(0xa00c);
1738                                                 }
1739                                                 if(lockobj == data2) {
1740                                                         lockresult = 0;
1741                                                         lockflag = true;
1742 #ifndef INTERRUPT
1743                                                         reside = false;
1744 #endif
1745                                                 } else {
1746                                                         // conflicts on lockresults
1747                                                         raw_test_done(0xa003);
1748                                                 }
1749                                                 break;
1750                                         }
1751                         case 5: {
1752                                                 // receive lock release msg
1753                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1754                                                         // no locks for this object, something is wrong
1755                                                         raw_test_done(0xa004);
1756                                                 } else {
1757                                                         int rwlock_obj = 0;
1758                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1759 #ifdef RAWDEBUG
1760                                                         raw_test_pass(0xe885);
1761                                                         raw_test_pass_reg(rwlock_obj);
1762 #endif
1763                                                         if(data1 == 0) {
1764                                                                 rwlock_obj--;
1765                                                         } else {
1766                                                                 rwlock_obj++;
1767                                                         }
1768                                                         RuntimeHashremovekey(locktbl, data2);
1769                                                         RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1770 #ifdef RAWDEBUG
1771                                                         raw_test_pass_reg(rwlock_obj);
1772 #endif
1773                                                 }
1774                                                 break;
1775                                         }
1776                         default:
1777                                         break;
1778                 }
1779                 //RUNFREE(msgdata);
1780                 //msgdata = NULL;
1781                 for(msgdataindex--;msgdataindex > 0; --msgdataindex) {
1782                         msgdata[msgdataindex] = -1;
1783                 }
1784                 msgtype = -1;
1785                 //msgdataindex = 0;
1786                 msglength = 30;
1787 #ifdef RAWDEBUG
1788                 raw_test_pass(0xe886);
1789 #endif
1790                 if(gdn_input_avail() != 0) {
1791                         goto msg;
1792                 }
1793                 return type;
1794         } else {
1795                 // not a whole msg
1796 #ifdef RAWDEBUG
1797                 raw_test_pass(0xe887);
1798 #endif
1799                 return -2;
1800         }
1801 #elif defined THREADSIMULATE
1802         int numofcore = pthread_getspecific(key);
1803         // use POSIX message queue to transfer object
1804         int msglen = 0;
1805         struct mq_attr mqattr;
1806         mq_getattr(mqd[numofcore], &mqattr);
1807         void * msgptr =RUNMALLOC(mqattr.mq_msgsize);
1808         msglen=mq_receive(mqd[numofcore], msgptr, mqattr.mq_msgsize, NULL); // receive the object into the queue
1809         if(-1 == msglen) {
1810                 // no msg
1811                 free(msgptr);
1812                 return 1;
1813         }
1814         //printf("msg: %s\n",msgptr);
1815         if(((int*)msgptr)[0] == -1) {
1816                 // StallMsg
1817                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1818                 int index = tmpptr->flag;
1819                 corestatus[index] = 0;
1820                 numsendobjs[index] = tmpptr->___cachedHash___;
1821                 numreceiveobjs[index] = tmpptr->___cachedCode___;
1822                 printf("<receiveObject> index: %d, sendobjs: %d, reveiveobjs: %d\n", index, numsendobjs[index], numreceiveobjs[index]);
1823                 free(msgptr);
1824                 return 2;
1825         } /*else if(((int*)msgptr)[0] == -2) {
1826                 // terminate msg
1827                 return 3;
1828         } */else {
1829                 // an object
1830                 if(numofcore == STARTUPCORE) {
1831                         ++(numreceiveobjs[numofcore]);
1832                 } else {
1833                         ++(thread_data_array[numofcore].numreceiveobjs);
1834                 }
1835                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1836                 struct transObjInfo * transObj = (struct transObjInfo *)tmpptr->original;
1837                 tmpptr = (struct ___Object___ *)(transObj->objptr);
1838                 int type = tmpptr->type;
1839                 int size=classsize[type];
1840                 struct ___Object___ * newobj=RUNMALLOC(size);
1841                 memcpy(newobj, tmpptr, size);
1842                 if(0 == newobj->isolate) {
1843                         newobj->original=tmpptr;
1844                 }
1845                 RUNFREE(msgptr);
1846                 tmpptr = NULL;
1847                 int k = 0;
1848                 for(k = 0; k < transObj->length; ++k) {
1849                         int taskindex = transObj->queues[2 * k];
1850                         int paramindex = transObj->queues[2 * k + 1];
1851                         struct parameterwrapper ** queues = &(paramqueues[numofcore][taskindex][paramindex]);
1852                         enqueueObject(newobj, queues, 1);
1853                 }
1854                 RUNFREE(transObj->queues);
1855                 RUNFREE(transObj);
1856                 return 0;
1857         }
1858 #endif
1859 }
1860
1861 bool getreadlock(void * ptr) {
1862 #ifdef RAW
1863         unsigned msgHdr;
1864         int self_y, self_x, target_y, target_x;
1865         int targetcore = ((int)ptr >> 5) % TOTALCORE;
1866         // for 32 bit machine, the size is always 4 words
1867         //int msgsize = sizeof(int) * 4;
1868         int msgsize = 4;
1869
1870         lockobj = (int)ptr;
1871         lockflag = false;
1872 #ifndef INTERRUPT
1873         reside = false;
1874 #endif
1875         lockresult = 0;
1876
1877         if(targetcore == corenum) {
1878                 // reside on this core
1879                 bool deny = false;
1880 #ifdef INTERRUPT
1881                 raw_user_interrupts_off();
1882 #endif
1883                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1884                         // no locks for this object
1885                         // first time to operate on this shared object
1886                         // create a lock for it
1887                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1888                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
1889                 } else {
1890                         int rwlock_obj = 0;
1891                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
1892                         if(-1 != rwlock_obj) {
1893                                 rwlock_obj++;
1894                                 RuntimeHashremovekey(locktbl, (int)ptr);
1895                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
1896                         } else {
1897                                 deny = true;
1898                         }
1899                 }
1900 #ifdef INTERRUPT
1901                 raw_user_interrupts_on();
1902 #endif
1903                 if(lockobj == (int)ptr) {
1904                         if(deny) {
1905                                 lockresult = 0;
1906                         } else {
1907                                 lockresult = 1;
1908                         }
1909                         lockflag = true;
1910 #ifndef INTERRUPT
1911                         reside = true;
1912 #endif
1913                 } else {
1914                         // conflicts on lockresults
1915                         raw_test_done(0xa005);
1916                 }
1917                 return true;
1918         }
1919
1920         calCoords(corenum, &self_y, &self_x);
1921         calCoords(targetcore, &target_y, &target_x);
1922         // Build the message header
1923     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1924                                            self_y, self_x,
1925                                                            target_y, target_x);
1926         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1927 #ifdef RAWDEBUG
1928         raw_test_pass(0xbbbb);
1929         raw_test_pass(0xb000 + targetcore); // targetcore
1930 #endif
1931     gdn_send(2); // lock request
1932  #ifdef RAWDEBUG
1933     raw_test_pass(2);
1934 #endif
1935         gdn_send(0); // read lock
1936 #ifdef RAWDEBUG
1937     raw_test_pass(0);
1938 #endif
1939     gdn_send(ptr);
1940 #ifdef RAWDEBUG
1941         raw_test_pass_reg(ptr);
1942 #endif
1943         gdn_send(corenum);
1944 #ifdef RAWDEBUG
1945         raw_test_pass_reg(corenum);
1946         raw_test_pass(0xffff);
1947 #endif
1948         return true;
1949 #elif defined THREADSIMULATE
1950         int numofcore = pthread_getspecific(key);
1951
1952         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
1953         printf("[getreadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1954         if(0 != rc) {
1955                 return false;
1956         }
1957         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1958                 // no locks for this object
1959                 // first time to operate on this shared object
1960                 // create a lock for it
1961                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1962                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1963                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
1964                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
1965                 rc = pthread_rwlock_init(rwlock, NULL);
1966                 printf("[getreadlock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1967                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
1968                 printf("[getreadlock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1969                 if(0 != rc) {
1970                         RUNFREE(rwlock);
1971                         return false;
1972                 } else {
1973                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1974                                 // check again
1975                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
1976                         } else {
1977                                 RUNFREE(rwlock);
1978                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
1979                         }
1980                         rc = pthread_rwlock_unlock(&rwlock_tbl);
1981                                 printf("[getreadlock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1982                 }
1983                 rc = pthread_rwlock_tryrdlock(rwlock);
1984                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1985                 if(0 != rc) {
1986                         return false;
1987                 } else {
1988                         return true;
1989                 }
1990         } else {
1991                 pthread_rwlock_t* rwlock_obj = NULL;
1992                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
1993                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1994                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1995                 int rc_obj = pthread_rwlock_tryrdlock(rwlock_obj);
1996                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
1997                 if(0 != rc_obj) {
1998                         return false;
1999                 } else {
2000                         return true;
2001                 }
2002         }
2003 #endif
2004 }
2005
2006 void releasereadlock(void * ptr) {
2007 #ifdef RAW
2008         unsigned msgHdr;
2009         int self_y, self_x, target_y, target_x;
2010         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2011         // for 32 bit machine, the size is always 3 words
2012         //int msgsize = sizeof(int) * 3;
2013         int msgsize = 3;
2014
2015         if(targetcore == corenum) {
2016 #ifdef INTERRUPT
2017                 raw_user_interrupts_off();
2018 #endif
2019                 // reside on this core
2020                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2021                         // no locks for this object, something is wrong
2022                         raw_test_done(0xa006);
2023                 } else {
2024                         int rwlock_obj = 0;
2025                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2026                         rwlock_obj--;
2027                         RuntimeHashremovekey(locktbl, (int)ptr);
2028                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2029                 }
2030 #ifdef INTERRUPT
2031                 raw_user_interrupts_on();
2032 #endif
2033                 return;
2034         }
2035
2036         calCoords(corenum, &self_y, &self_x);
2037         calCoords(targetcore, &target_y, &target_x);
2038         // Build the message header
2039     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2040                                            self_y, self_x,
2041                                                            target_y, target_x);
2042         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2043 #ifdef RAWDEBUG
2044         raw_test_pass(0xbbbb);
2045         raw_test_pass(0xb000 + targetcore); // targetcore
2046 #endif
2047     gdn_send(5); // lock release
2048 #ifdef RAWDEBUG
2049     raw_test_pass(5);
2050 #endif
2051         gdn_send(0); // read lock
2052 #ifdef RAWDEBUG
2053     raw_test_pass(0);
2054 #endif
2055     gdn_send(ptr);
2056 #ifdef RAWDEBUG
2057         raw_test_pass_reg(ptr);
2058         raw_test_pass(0xffff);
2059 #endif
2060 #elif defined THREADSIMULATE
2061         int numofcore = pthread_getspecific(key);
2062         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
2063         printf("[releasereadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2064         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2065                 printf("[releasereadlock, %d] Error: try to release a lock without previously grab it\n", numofcore);
2066                 exit(-1);
2067         }
2068         pthread_rwlock_t* rwlock_obj = NULL;
2069         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2070         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
2071         printf("[releasereadlock, %d] unlocked object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2072         rc = pthread_rwlock_unlock(&rwlock_tbl);
2073         printf("[releasereadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2074 #endif
2075 }
2076
2077 #ifdef RAW
2078 bool getreadlock_I(void * ptr) {
2079         unsigned msgHdr;
2080         int self_y, self_x, target_y, target_x;
2081         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2082         // for 32 bit machine, the size is always 4 words
2083         //int msgsize = sizeof(int) * 4;
2084         int msgsize = 4;
2085
2086         lockobj = (int)ptr;
2087         lockflag = false;
2088 #ifndef INTERRUPT
2089         reside = false;
2090 #endif
2091         lockresult = 0;
2092
2093         if(targetcore == corenum) {
2094                 // reside on this core
2095                 bool deny = false;
2096                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2097                         // no locks for this object
2098                         // first time to operate on this shared object
2099                         // create a lock for it
2100                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
2101                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
2102                 } else {
2103                         int rwlock_obj = 0;
2104                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2105                         if(-1 != rwlock_obj) {
2106                                 rwlock_obj++;
2107                                 RuntimeHashremovekey(locktbl, (int)ptr);
2108                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2109                         } else {
2110                                 deny = true;
2111                         }
2112                 }
2113                 if(lockobj == (int)ptr) {
2114                         if(deny) {
2115                                 lockresult = 0;
2116                         } else {
2117                                 lockresult = 1;
2118                         }
2119                         lockflag = true;
2120 #ifndef INTERRUPT
2121                         reside = true;
2122 #endif
2123                 } else {
2124                         // conflicts on lockresults
2125                         raw_test_done(0xa005);
2126                 }
2127                 return true;
2128         }
2129
2130         calCoords(corenum, &self_y, &self_x);
2131         calCoords(targetcore, &target_y, &target_x);
2132         // Build the message header
2133     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2134                                            self_y, self_x,
2135                                                            target_y, target_x);
2136         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2137 #ifdef RAWDEBUG
2138         raw_test_pass(0xbbbb);
2139         raw_test_pass(0xb000 + targetcore); // targetcore
2140 #endif
2141     gdn_send(2); // lock request
2142 #ifdef RAWDEBUG
2143     raw_test_pass(2);
2144 #endif
2145         gdn_send(0); // read lock
2146 #ifdef RAWDEBUG
2147     raw_test_pass(0);
2148 #endif
2149     gdn_send(ptr);
2150 #ifdef RAWDEBUG
2151         raw_test_pass_reg(ptr);
2152 #endif
2153         gdn_send(corenum);
2154 #ifdef RAWDEBUG
2155         raw_test_pass_reg(corenum);
2156         raw_test_pass(0xffff);
2157 #endif
2158         return true;
2159 }
2160
2161 void releasereadlock_I(void * ptr) {
2162         unsigned msgHdr;
2163         int self_y, self_x, target_y, target_x;
2164         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2165         // for 32 bit machine, the size is always 3 words
2166         //int msgsize = sizeof(int) * 3;
2167         int msgsize = 3;
2168
2169         if(targetcore == corenum) {
2170                 // reside on this core
2171                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2172                         // no locks for this object, something is wrong
2173                         raw_test_done(0xa006);
2174                 } else {
2175                         int rwlock_obj = 0;
2176                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2177                         rwlock_obj--;
2178                         RuntimeHashremovekey(locktbl, (int)ptr);
2179                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2180                 }
2181                 return;
2182         }
2183
2184         calCoords(corenum, &self_y, &self_x);
2185         calCoords(targetcore, &target_y, &target_x);
2186         // Build the message header
2187     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2188                                            self_y, self_x,
2189                                                            target_y, target_x);
2190         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2191 #ifdef RAWDEBUG
2192         raw_test_pass(0xbbbb);
2193         raw_test_pass(0xb000 + targetcore); // targetcore
2194 #endif
2195     gdn_send(5); // lock release
2196 #ifdef RAWDEBUG
2197     raw_test_pass(5);
2198 #endif
2199         gdn_send(0); // read lock
2200 #ifdef RAWDEBUG
2201     raw_test_pass(0);
2202 #endif
2203     gdn_send(ptr);
2204 #ifdef RAWDEBUG
2205         raw_test_pass_reg(ptr);
2206         raw_test_pass(0xffff);
2207 #endif
2208 }
2209 #endif
2210
2211 // not reentrant
2212 bool getwritelock(void * ptr) {
2213 #ifdef RAW
2214         unsigned msgHdr;
2215         int self_y, self_x, target_y, target_x;
2216         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2217         // for 32 bit machine, the size is always 4 words
2218         //int msgsize = sizeof(int) * 4;
2219         int msgsize= 4;
2220         int tc = TOTALCORE;
2221 #ifdef INTERRUPT
2222         //raw_user_interrupts_off();
2223 #endif
2224         //targetcore = ((int)ptr) % tc;
2225 #ifdef INTERRUPT
2226         //raw_user_interrupts_on();
2227 #endif
2228
2229 #ifdef RAWDEBUG
2230         raw_test_pass(0xe551);
2231         raw_test_pass_reg(ptr);
2232         raw_test_pass_reg(targetcore);
2233         raw_test_pass_reg(tc);
2234 #endif
2235
2236         lockobj = (int)ptr;
2237         lockflag = false;
2238 #ifndef INTERRUPT
2239         reside = false;
2240 #endif
2241         lockresult = 0;
2242
2243         if(targetcore == corenum) {
2244                 // reside on this core
2245                 bool deny = false;
2246 #ifdef INTERRUPT
2247                 raw_user_interrupts_off();
2248 #endif
2249                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2250                         // no locks for this object
2251                         // first time to operate on this shared object
2252                         // create a lock for it
2253                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
2254 #ifdef RAWDEBUG
2255                         raw_test_pass(0xe552);
2256 #endif
2257                         RuntimeHashadd_I(locktbl, (int)ptr, -1);
2258                 } else {
2259                         int rwlock_obj = 0;
2260                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2261 #ifdef RAWDEBUG
2262                         raw_test_pass(0xe553);
2263                         raw_test_pass_reg(rwlock_obj);
2264 #endif
2265                         if(0 == rwlock_obj) {
2266                                 rwlock_obj = -1;
2267                                 RuntimeHashremovekey(locktbl, (int)ptr);
2268                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2269                         } else {
2270                                 deny = true;
2271                         }
2272                 }
2273 #ifdef INTERRUPT
2274                 raw_user_interrupts_on();
2275 #endif
2276 #ifdef RAWDEBUG
2277                 raw_test_pass(0xe554);
2278                 raw_test_pass_reg(lockresult);
2279 #endif
2280                 if(lockobj == (int)ptr) {
2281                         if(deny) {
2282                                 lockresult = 0;
2283 #ifdef RAWDEBUG
2284                                 raw_test_pass(0);
2285 #endif
2286                         } else {
2287                                 lockresult = 1;
2288 #ifdef RAWDEBUG
2289                 raw_test_pass(1);
2290 #endif
2291                         }
2292                         lockflag = true;
2293 #ifndef INTERRUPT
2294                         reside = true;
2295 #endif
2296                 } else {
2297                         // conflicts on lockresults
2298                         raw_test_done(0xa007);
2299                 }
2300                 return true;
2301         }
2302
2303 #ifdef RAWDEBUG
2304         raw_test_pass(0xe555);
2305 #endif
2306         calCoords(corenum, &self_y, &self_x);
2307         calCoords(targetcore, &target_y, &target_x);
2308         // Build the message header
2309     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2310                                            self_y, self_x,
2311                                                            target_y, target_x);
2312         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2313 #ifdef RAWDEBUG
2314         raw_test_pass(0xbbbb);
2315         raw_test_pass(0xb000 + targetcore); // targetcore
2316 #endif
2317     gdn_send(2); // lock request
2318 #ifdef RAWDEBUG
2319     raw_test_pass(2);
2320 #endif
2321         gdn_send(1); // write lock
2322 #ifdef RAWDEBUG
2323     raw_test_pass(1);
2324 #endif
2325     gdn_send(ptr);
2326 #ifdef RAWDEBUG
2327         raw_test_pass_reg(ptr);
2328 #endif
2329         gdn_send(corenum);
2330 #ifdef RAWDEBUG
2331         raw_test_pass_reg(corenum);
2332         raw_test_pass(0xffff);
2333 #endif
2334         return true;
2335 #elif defined THREADSIMULATE
2336         int numofcore = pthread_getspecific(key);
2337
2338         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
2339         printf("[getwritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2340         if(0 != rc) {
2341                 return false;
2342         }
2343         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2344                 // no locks for this object
2345                 // first time to operate on this shared object
2346                 // create a lock for it
2347                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2348                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2349                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
2350                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
2351                 rc = pthread_rwlock_init(rwlock, NULL);
2352                 printf("[getwritelock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2353                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
2354                 printf("[getwritelock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2355                 if(0 != rc) {
2356                         pthread_rwlock_destroy(rwlock);
2357                         RUNFREE(rwlock);
2358                         return false;
2359                 } else {
2360                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2361                                 // check again
2362                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
2363                         } else {
2364                                 pthread_rwlock_destroy(rwlock);
2365                                 RUNFREE(rwlock);
2366                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
2367                         }
2368                         rc = pthread_rwlock_unlock(&rwlock_tbl);
2369                         printf("[getwritelock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2370                 }
2371                 rc = pthread_rwlock_trywrlock(rwlock);
2372                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2373                 if(0 != rc) {
2374                         return false;
2375                 } else {
2376                         return true;
2377                 }
2378         } else {
2379                 pthread_rwlock_t* rwlock_obj = NULL;
2380                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2381                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2382                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2383                 int rc_obj = pthread_rwlock_trywrlock(rwlock_obj);
2384                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2385                 if(0 != rc_obj) {
2386                         return false;
2387                 } else {
2388                         return true;
2389                 }
2390         }
2391
2392 #endif
2393 }
2394
2395 void releasewritelock(void * ptr) {
2396 #ifdef RAW
2397         unsigned msgHdr;
2398         int self_y, self_x, target_y, target_x;
2399         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2400         // for 32 bit machine, the size is always 3 words
2401         //int msgsize = sizeof(int) * 3;
2402         int msgsize = 3;
2403
2404         if(targetcore == corenum) {
2405 #ifdef INTERRUPT
2406                 raw_user_interrupts_off();
2407 #endif
2408                 // reside on this core
2409                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2410                         // no locks for this object, something is wrong
2411                         raw_test_done(0xa008);
2412                 } else {
2413                         int rwlock_obj = 0;
2414 #ifdef RAWDEBUG
2415                         raw_test_pass(0xe662);
2416 #endif
2417                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2418 #ifdef RAWDEBUG
2419                         raw_test_pass_reg(rwlock_obj);
2420 #endif
2421                         rwlock_obj++;
2422                         RuntimeHashremovekey(locktbl, (int)ptr);
2423                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2424 #ifdef RAWDEBUG
2425                         raw_test_pass_reg(rwlock_obj);
2426 #endif
2427                 }
2428 #ifdef INTERRUPT
2429                 raw_user_interrupts_on();
2430 #endif
2431                 return;
2432         }
2433
2434 #ifdef RAWDEBUG
2435         raw_test_pass(0xe663);
2436 #endif
2437         calCoords(corenum, &self_y, &self_x);
2438         calCoords(targetcore, &target_y, &target_x);
2439         // Build the message header
2440     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2441                                            self_y, self_x,
2442                                                            target_y, target_x);
2443         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2444 #ifdef RAWDEBUG
2445         raw_test_pass(0xbbbb);
2446         raw_test_pass(0xb000 + targetcore);
2447 #endif
2448     gdn_send(5); // lock release
2449  #ifdef RAWDEBUG
2450     raw_test_pass(5);
2451 #endif
2452         gdn_send(1); // write lock
2453 #ifdef RAWDEBUG
2454     raw_test_pass(1);
2455 #endif
2456     gdn_send(ptr);
2457 #ifdef RAWDEBUG
2458         raw_test_pass_reg(ptr);
2459         raw_test_pass(0xffff);
2460 #endif
2461 #elif defined THREADSIMULATE
2462         int numofcore = pthread_getspecific(key);
2463         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
2464         printf("[releasewritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2465         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2466                 printf("[releasewritelock, %d] Error: try to release a lock without previously grab it\n", numofcore);
2467                 exit(-1);
2468         }
2469         pthread_rwlock_t* rwlock_obj = NULL;
2470         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2471         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
2472         printf("[releasewritelock, %d] unlocked object %d: %d error:\n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2473         rc = pthread_rwlock_unlock(&rwlock_tbl);
2474         printf("[releasewritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2475 #endif
2476 }
2477
2478 int enqueuetasks(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2479   void * taskpointerarray[MAXTASKPARAMS];
2480   int j;
2481   int numparams=parameter->task->numParameters;
2482   int numiterators=parameter->task->numTotal-1;
2483   int retval=1;
2484   int addnormal=1;
2485   int adderror=1;
2486
2487   struct taskdescriptor * task=parameter->task;
2488
2489         ObjectHashadd(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2490
2491   /* Add enqueued object to parameter vector */
2492   taskpointerarray[parameter->slot]=ptr;
2493
2494   /* Reset iterators */
2495   for(j=0;j<numiterators;j++) {
2496     toiReset(&parameter->iterators[j]);
2497   }
2498
2499   /* Find initial state */
2500   for(j=0;j<numiterators;j++) {
2501   backtrackinit:
2502     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2503       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2504     else if (j>0) {
2505       /* Need to backtrack */
2506       toiReset(&parameter->iterators[j]);
2507       j--;
2508       goto backtrackinit;
2509     } else {
2510       /* Nothing to enqueue */
2511       return retval;
2512     }
2513   }
2514
2515
2516   while(1) {
2517     /* Enqueue current state */
2518     int launch = 0;
2519     struct taskparamdescriptor *tpd=RUNMALLOC(sizeof(struct taskparamdescriptor));
2520     tpd->task=task;
2521     tpd->numParameters=numiterators+1;
2522     tpd->parameterArray=RUNMALLOC(sizeof(void *)*(numiterators+1));
2523     for(j=0;j<=numiterators;j++){
2524       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2525     }
2526     /* Enqueue task */
2527     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2528       genputtable(activetasks, tpd, tpd);
2529     } else {
2530       RUNFREE(tpd->parameterArray);
2531       RUNFREE(tpd);
2532     }
2533
2534     /* This loop iterates to the next parameter combination */
2535     if (numiterators==0)
2536       return retval;
2537
2538     for(j=numiterators-1; j<numiterators;j++) {
2539     backtrackinc:
2540       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2541         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2542       else if (j>0) {
2543         /* Need to backtrack */
2544         toiReset(&parameter->iterators[j]);
2545         j--;
2546         goto backtrackinc;
2547       } else {
2548         /* Nothing more to enqueue */
2549         return retval;
2550       }
2551     }
2552   }
2553   return retval;
2554 }
2555
2556 #ifdef RAW
2557 int enqueuetasks_I(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2558   void * taskpointerarray[MAXTASKPARAMS];
2559   int j;
2560   int numparams=parameter->task->numParameters;
2561   int numiterators=parameter->task->numTotal-1;
2562   int retval=1;
2563   int addnormal=1;
2564   int adderror=1;
2565
2566   struct taskdescriptor * task=parameter->task;
2567
2568         ObjectHashadd_I(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2569
2570   /* Add enqueued object to parameter vector */
2571   taskpointerarray[parameter->slot]=ptr;
2572
2573   /* Reset iterators */
2574   for(j=0;j<numiterators;j++) {
2575     toiReset(&parameter->iterators[j]);
2576   }
2577
2578   /* Find initial state */
2579   for(j=0;j<numiterators;j++) {
2580   backtrackinit:
2581     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2582       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2583     else if (j>0) {
2584       /* Need to backtrack */
2585       toiReset(&parameter->iterators[j]);
2586       j--;
2587       goto backtrackinit;
2588     } else {
2589       /* Nothing to enqueue */
2590       return retval;
2591     }
2592   }
2593
2594   while(1) {
2595     /* Enqueue current state */
2596     int launch = 0;
2597     struct taskparamdescriptor *tpd=RUNMALLOC_I(sizeof(struct taskparamdescriptor));
2598     tpd->task=task;
2599     tpd->numParameters=numiterators+1;
2600     tpd->parameterArray=RUNMALLOC_I(sizeof(void *)*(numiterators+1));
2601     for(j=0;j<=numiterators;j++){
2602       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2603     }
2604     /* Enqueue task */
2605     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2606       genputtable_I(activetasks, tpd, tpd);
2607     } else {
2608       RUNFREE(tpd->parameterArray);
2609       RUNFREE(tpd);
2610     }
2611
2612     /* This loop iterates to the next parameter combination */
2613     if (numiterators==0)
2614       return retval;
2615
2616     for(j=numiterators-1; j<numiterators;j++) {
2617     backtrackinc:
2618       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2619         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2620       else if (j>0) {
2621         /* Need to backtrack */
2622         toiReset(&parameter->iterators[j]);
2623         j--;
2624         goto backtrackinc;
2625       } else {
2626         /* Nothing more to enqueue */
2627         return retval;
2628       }
2629     }
2630   }
2631   return retval;
2632 }
2633 #endif
2634
2635 /* Handler for signals. The signals catch null pointer errors and
2636    arithmatic errors. */
2637 #ifndef RAW
2638 void myhandler(int sig, siginfo_t *info, void *uap) {
2639   sigset_t toclear;
2640 #ifdef DEBUG
2641   printf("sig=%d\n",sig);
2642   printf("signal\n");
2643 #endif
2644   sigemptyset(&toclear);
2645   sigaddset(&toclear, sig);
2646   sigprocmask(SIG_UNBLOCK, &toclear,NULL);
2647   longjmp(error_handler,1);
2648 }
2649 #endif
2650
2651 fd_set readfds;
2652 int maxreadfd;
2653 struct RuntimeHash *fdtoobject;
2654
2655 void addreadfd(int fd) {
2656   if (fd>=maxreadfd)
2657     maxreadfd=fd+1;
2658   FD_SET(fd, &readfds);
2659 }
2660
2661 void removereadfd(int fd) {
2662   FD_CLR(fd, &readfds);
2663   if (maxreadfd==(fd+1)) {
2664     maxreadfd--;
2665     while(maxreadfd>0&&!FD_ISSET(maxreadfd-1, &readfds))
2666       maxreadfd--;
2667   }
2668 }
2669
2670 #ifdef PRECISE_GC
2671 #define OFFSET 2
2672 #else
2673 #define OFFSET 0
2674 #endif
2675
2676 void executetasks() {
2677   void * taskpointerarray[MAXTASKPARAMS+OFFSET];
2678   int numparams=0;
2679   int numtotal=0;
2680   struct ___Object___ * tmpparam = NULL;
2681   struct parameterdescriptor * pd=NULL;
2682   struct parameterwrapper *pw=NULL;
2683   int j = 0;
2684   int x = 0;
2685   bool lock = true;
2686
2687 #ifdef RAW
2688   int grount = 0;
2689   int andmask=0;
2690   int checkmask=0;
2691 #ifdef RAWDEBUG
2692   raw_test_pass(0xe991);
2693 #endif
2694 #endif
2695
2696 #ifndef RAW
2697   /* Set up signal handlers */
2698   struct sigaction sig;
2699   sig.sa_sigaction=&myhandler;
2700   sig.sa_flags=SA_SIGINFO;
2701   sigemptyset(&sig.sa_mask);
2702
2703   /* Catch bus errors, segmentation faults, and floating point exceptions*/
2704   sigaction(SIGBUS,&sig,0);
2705   sigaction(SIGSEGV,&sig,0);
2706   sigaction(SIGFPE,&sig,0);
2707   sigaction(SIGPIPE,&sig,0);
2708 #endif
2709
2710 #ifndef RAW
2711   /* Zero fd set */
2712   FD_ZERO(&readfds);
2713 #endif
2714   maxreadfd=0;
2715 #ifndef RAW
2716   fdtoobject=allocateRuntimeHash(100);
2717 #endif
2718
2719 #ifndef RAW
2720   /* Map first block of memory to protected, anonymous page */
2721   mmap(0, 0x1000, 0, MAP_SHARED|MAP_FIXED|MAP_ANON, -1, 0);
2722 #endif
2723
2724   newtask:
2725   while((hashsize(activetasks)>0)||(maxreadfd>0)) {
2726
2727 #ifdef RAW
2728 #ifdef RAWDEBUG
2729           raw_test_pass(0xe992);
2730 #endif
2731 #else
2732     /* Check if any filedescriptors have IO pending */
2733     if (maxreadfd>0) {
2734       int i;
2735       struct timeval timeout={0,0};
2736       fd_set tmpreadfds;
2737       int numselect;
2738       tmpreadfds=readfds;
2739       numselect=select(maxreadfd, &tmpreadfds, NULL, NULL, &timeout);
2740       if (numselect>0) {
2741         /* Process ready fd's */
2742         int fd;
2743         for(fd=0;fd<maxreadfd;fd++) {
2744           if (FD_ISSET(fd, &tmpreadfds)) {
2745             /* Set ready flag on object */
2746             void * objptr;
2747             //      printf("Setting fd %d\n",fd);
2748             if (RuntimeHashget(fdtoobject, fd,(int *) &objptr)) {
2749               if(intflagorand(objptr,1,0xFFFFFFFF)) { /* Set the first flag to 1 */
2750                          enqueueObject(objptr, NULL, 0);
2751                   }
2752             }
2753           }
2754         }
2755       }
2756     }
2757 #endif
2758
2759     /* See if there are any active tasks */
2760     if (hashsize(activetasks)>0) {
2761       int i;
2762       currtpd=(struct taskparamdescriptor *) getfirstkey(activetasks);
2763       genfreekey(activetasks, currtpd);
2764
2765       /* Check if this task has failed, allow a task that contains optional objects to fire */
2766       /*if (gencontains(failedtasks, currtpd)) {
2767         // Free up task parameter descriptor
2768         RUNFREE(currtpd->parameterArray);
2769         RUNFREE(currtpd);
2770         goto newtask;
2771       }*/
2772       numparams=currtpd->task->numParameters;
2773       numtotal=currtpd->task->numTotal;
2774
2775 #ifdef THREADSIMULATE
2776           int isolateflags[numparams];
2777 #endif
2778       /* Make sure that the parameters are still in the queues */
2779       for(i=0;i<numparams;i++) {
2780         void * parameter=currtpd->parameterArray[i];
2781 #ifdef RAW
2782 #ifdef RAWDEBUG
2783         raw_test_pass(0xe993);
2784 #endif
2785
2786         if(((struct ___Object___ *)parameter)->type == STARTUPTYPE) {
2787                         lock = false;
2788         taskpointerarray[i+OFFSET]=parameter;
2789         goto execute;
2790         }
2791                 lock = true;
2792         // require locks for this parameter if it is not a startup object
2793         getwritelock(parameter);
2794         grount = 0;
2795
2796 #ifdef INTERRUPT
2797         raw_user_interrupts_off();
2798 #endif
2799         while(!lockflag) {
2800                 receiveObject();
2801         }
2802 #ifndef INTERRUPT
2803         if(reside) {
2804                 while(receiveObject() != -1) {
2805                 }
2806         }
2807 #endif
2808         grount = lockresult;
2809
2810         lockresult = 0;
2811         lockobj = 0;
2812         lockflag = false;
2813 #ifndef INTERRUPT
2814         reside = false;
2815 #endif
2816 #ifdef INTERRUPT
2817         raw_user_interrupts_on();
2818 #endif
2819
2820         if(grount == 0) {
2821 #ifdef RAWDEBUG
2822                 raw_test_pass(0xe994);
2823 #endif
2824                 // can not get the lock, try later
2825                 for(j = 0; j < i; ++j) {
2826                         releasewritelock(taskpointerarray[j+OFFSET]);
2827                 }
2828                 genputtable(activetasks, currtpd, currtpd);
2829                 if(hashsize(activetasks) == 1) {
2830                         // only one task right now, wait a little while before next try
2831                         int halt = 10000;
2832                         while(halt--){}
2833                 }
2834                 goto newtask;
2835         }
2836         // flush the object
2837         {
2838                 raw_invalidate_cache_range((int)parameter, classsize[((struct ___Object___ *)parameter)->type]);
2839                 /*int tmp = 0;
2840                 for(tmp = 0; tmp < classsize[((struct ___Object___ *)parameter)->type]; ++tmp) {
2841                         invalidateAddr(parameter + tmp);
2842                 }*/
2843         }
2844 #endif
2845         tmpparam = (struct ___Object___ *)parameter;
2846 #ifdef THREADSIMULATE
2847         if(((struct ___Object___ *)parameter)->type == STARTUPTYPE) {
2848                         lock = false;
2849         taskpointerarray[i+OFFSET]=parameter;
2850         goto execute;
2851         }
2852         lock = true;
2853         if(0 == tmpparam->isolate) {
2854                 isolateflags[i] = 0;
2855                 // shared object, need to flush with current value
2856                 //if(!getreadlock(tmpparam->original)) {
2857                 //      // fail to get read lock of the original object, try this task later
2858                 if(!getwritelock(tmpparam->original)) {
2859                         // fail to get write lock, release all obtained locks and try this task later
2860                         int j = 0;
2861                         for(j = 0; j < i; ++j) {
2862                                 if(0 == isolateflags[j]) {
2863                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2864                                 }
2865                         }
2866                         genputtable(activetasks, currtpd, currtpd);
2867                         goto newtask;
2868                 }
2869                 if(tmpparam->version != tmpparam->original->version) {
2870                         // some task on another core has changed this object
2871                         // flush this object
2872                         //memcpy(tmpparam, tmpparam->original, classsize[tmpparam->type]);
2873                         // release all obtained locks
2874                         int j = 0;
2875                         for(j = 0; j < i; ++j) {
2876                                 if(0 == isolateflags[j]) {
2877                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2878                                 }
2879                         }
2880                         releasewritelock(tmpparam->original);
2881
2882                         // dequeue this object
2883                         int numofcore = pthread_getspecific(key);
2884                         struct parameterwrapper ** queues = objectqueues[numofcore][tmpparam->type];
2885                         int length = numqueues[numofcore][tmpparam->type];
2886                         for(j = 0; j < length; ++j) {
2887                                 struct parameterwrapper * pw = queues[j];
2888                                 if(ObjectHashcontainskey(pw->objectset, (int)tmpparam)) {
2889                                         int next;
2890                                         int UNUSED, UNUSED2;
2891                                         int * enterflags;
2892                                         ObjectHashget(pw->objectset, (int) tmpparam, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2893                                         ObjectHashremove(pw->objectset, (int)tmpparam);
2894                                         if (enterflags!=NULL)
2895                                                 free(enterflags);
2896                                 }
2897                         }
2898                         // try to enqueue it again to check if it feeds other tasks;
2899                         //enqueueObject(tmpparam, NULL, 0);
2900                         // Free up task parameter descriptor
2901                         RUNFREE(currtpd->parameterArray);
2902                         RUNFREE(currtpd);
2903                         goto newtask;
2904                 }
2905         } else {
2906                 isolateflags[i] = 1;
2907         }
2908 #endif
2909         pd=currtpd->task->descriptorarray[i];
2910         pw=(struct parameterwrapper *) pd->queue;
2911         /* Check that object is still in queue */
2912         {
2913           if (!ObjectHashcontainskey(pw->objectset, (int) parameter)) {
2914 #ifdef RAWDEBUG
2915                   raw_test_pass(0xe995);
2916 #endif
2917                 // release grabbed locks
2918                 for(j = 0; j < i; ++j) {
2919                         releasewritelock(taskpointerarray[j+OFFSET]);
2920                 }
2921                 releasewritelock(parameter);
2922             RUNFREE(currtpd->parameterArray);
2923             RUNFREE(currtpd);
2924             goto newtask;
2925           }
2926         }
2927 #ifdef RAW
2928         /* Check if the object's flags still meets requirements */
2929         {
2930                 int tmpi = 0;
2931                 bool ismet = false;
2932                 for(tmpi = 0; tmpi < pw->numberofterms; ++tmpi) {
2933                         andmask=pw->intarray[tmpi*2];
2934                         checkmask=pw->intarray[tmpi*2+1];
2935 #ifdef RAWDEBUG
2936                         raw_test_pass(0xdd000000 + andmask);
2937                         raw_test_pass_reg((int)parameter);
2938                         raw_test_pass(0xdd000000 + ((struct ___Object___ *)parameter)->flag);
2939                         raw_test_pass(0xdd000000 + checkmask);
2940 #endif
2941                         if((((struct ___Object___ *)parameter)->flag&andmask)==checkmask) {
2942                                 ismet = true;
2943                                 break;
2944                         }
2945                 }
2946                 if (!ismet) {
2947                         // flags are never suitable
2948                         // remove this obj from the queue
2949                         int next;
2950                         int UNUSED, UNUSED2;
2951                         int * enterflags;
2952 #ifdef RAWDEBUG
2953                         raw_test_pass(0xe996);
2954 #endif
2955                         ObjectHashget(pw->objectset, (int) parameter, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2956                         ObjectHashremove(pw->objectset, (int)parameter);
2957                         if (enterflags!=NULL)
2958                                 free(enterflags);
2959                         // release grabbed locks
2960                         for(j = 0; j < i; ++j) {
2961                                 releasewritelock(taskpointerarray[j+OFFSET]);
2962                         }
2963                         releasewritelock(parameter);
2964                         RUNFREE(currtpd->parameterArray);
2965                         RUNFREE(currtpd);
2966                         goto newtask;
2967                 }
2968         }
2969 #endif
2970       parameterpresent:
2971         ;
2972         /* Check that object still has necessary tags */
2973         for(j=0;j<pd->numbertags;j++) {
2974           int slotid=pd->tagarray[2*j]+numparams;
2975           struct ___TagDescriptor___ *tagd=currtpd->parameterArray[slotid];
2976           if (!containstag(parameter, tagd)) {
2977 #ifdef RAWDEBUG
2978                   raw_test_pass(0xe997);
2979 #endif
2980             RUNFREE(currtpd->parameterArray);
2981             RUNFREE(currtpd);
2982             goto newtask;
2983           }
2984         }
2985
2986         taskpointerarray[i+OFFSET]=parameter;
2987       }
2988       /* Copy the tags */
2989       for(;i<numtotal;i++) {
2990         taskpointerarray[i+OFFSET]=currtpd->parameterArray[i];
2991       }
2992
2993 #ifdef THREADSIMULATE
2994          for(i = 0; i < numparams; ++i) {
2995                   if(0 == isolateflags[i]) {
2996                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
2997                           if(tmpparam != tmpparam->original) {
2998                                   taskpointerarray[i+OFFSET] = tmpparam->original;
2999                           }
3000                   }
3001           }
3002 #endif
3003
3004       {
3005 #if 0
3006 #ifndef RAW
3007         /* Checkpoint the state */
3008         forward=allocateRuntimeHash(100);
3009         reverse=allocateRuntimeHash(100);
3010         //void ** checkpoint=makecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, forward, reverse);
3011 #endif
3012 #endif
3013         if (x=setjmp(error_handler)) {
3014           int counter;
3015           /* Recover */
3016 #ifndef RAW
3017 #ifdef DEBUG
3018           printf("Fatal Error=%d, Recovering!\n",x);
3019 #endif
3020 #endif
3021          /*
3022           genputtable(failedtasks,currtpd,currtpd);
3023           //restorecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, checkpoint, forward, reverse);
3024
3025           freeRuntimeHash(forward);
3026           freeRuntimeHash(reverse);
3027           freemalloc();
3028           forward=NULL;
3029           reverse=NULL;
3030           */
3031           //fflush(stdout);
3032 #ifdef RAW
3033 #ifdef RAWDEBUG
3034           raw_test_pass_reg(x);
3035 #endif
3036           raw_test_done(0xa009);
3037 #else
3038           exit(-1);
3039 #endif
3040         } else {
3041           /*if (injectfailures) {
3042             if ((((double)random())/RAND_MAX)<failurechance) {
3043               printf("\nINJECTING TASK FAILURE to %s\n", currtpd->task->name);
3044               longjmp(error_handler,10);
3045             }
3046           }*/
3047           /* Actually call task */
3048 #ifdef PRECISE_GC
3049           ((int *)taskpointerarray)[0]=currtpd->numParameters;
3050           taskpointerarray[1]=NULL;
3051 #endif
3052 execute:
3053           if(debugtask){
3054 #ifndef RAW
3055             printf("ENTER %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
3056 #endif
3057             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
3058 #ifndef RAW
3059             printf("EXIT %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
3060 #endif
3061           } else {
3062             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
3063           }
3064 #ifdef RAWDEBUG
3065           raw_test_pass(0xe998);
3066                  raw_test_pass_reg(lock);
3067                   #endif
3068
3069           if(lock) {
3070 #ifdef RAW
3071            for(i = 0; i < numparams; ++i) {
3072                   int j = 0;
3073                   struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
3074 #ifdef RAWDEBUG
3075                   raw_test_pass(0xe999);
3076                 raw_test_pass(0xdd100000 + tmpparam->flag);
3077 #endif
3078                   releasewritelock(tmpparam);
3079           }
3080 #elif defined THREADSIMULATE
3081           for(i = 0; i < numparams; ++i) {
3082                   if(0 == isolateflags[i]) {
3083                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
3084                           releasewritelock(tmpparam);
3085                   }
3086           }
3087 #endif
3088           }
3089
3090 #if 0
3091 #ifndef RAW
3092           freeRuntimeHash(forward);
3093           freeRuntimeHash(reverse);
3094 #endif
3095 #endif
3096           freemalloc();
3097           // Free up task parameter descriptor
3098           RUNFREE(currtpd->parameterArray);
3099           RUNFREE(currtpd);
3100 #if 0
3101 #ifndef RAW
3102           forward=NULL;
3103           reverse=NULL;
3104 #endif
3105 #endif
3106 #ifdef RAWDEBUG
3107           raw_test_pass(0xe99a);
3108                   raw_test_pass_reg(lock);
3109                   #endif
3110
3111         }
3112       }
3113     }
3114   }
3115 #ifdef RAWDEBUG
3116         raw_test_pass(0xe999);
3117 #endif
3118 }
3119
3120 /* This function processes an objects tags */
3121 void processtags(struct parameterdescriptor *pd, int index, struct parameterwrapper *parameter, int * iteratorcount, int *statusarray, int numparams) {
3122   int i;
3123
3124   for(i=0;i<pd->numbertags;i++) {
3125     int slotid=pd->tagarray[2*i];
3126     int tagid=pd->tagarray[2*i+1];
3127
3128     if (statusarray[slotid+numparams]==0) {
3129       parameter->iterators[*iteratorcount].istag=1;
3130       parameter->iterators[*iteratorcount].tagid=tagid;
3131       parameter->iterators[*iteratorcount].slot=slotid+numparams;
3132       parameter->iterators[*iteratorcount].tagobjectslot=index;
3133       statusarray[slotid+numparams]=1;
3134       (*iteratorcount)++;
3135     }
3136   }
3137 }
3138
3139
3140 void processobject(struct parameterwrapper *parameter, int index, struct parameterdescriptor *pd, int *iteratorcount, int * statusarray, int numparams) {
3141   int i;
3142   int tagcount=0;
3143   struct ObjectHash * objectset=((struct parameterwrapper *)pd->queue)->objectset;
3144
3145   parameter->iterators[*iteratorcount].istag=0;
3146   parameter->iterators[*iteratorcount].slot=index;
3147   parameter->iterators[*iteratorcount].objectset=objectset;
3148   statusarray[index]=1;
3149
3150   for(i=0;i<pd->numbertags;i++) {
3151     int slotid=pd->tagarray[2*i];
3152     int tagid=pd->tagarray[2*i+1];
3153     if (statusarray[slotid+numparams]!=0) {
3154       /* This tag has already been enqueued, use it to narrow search */
3155       parameter->iterators[*iteratorcount].tagbindings[tagcount]=slotid+numparams;
3156       tagcount++;
3157     }
3158   }
3159   parameter->iterators[*iteratorcount].numtags=tagcount;
3160
3161   (*iteratorcount)++;
3162 }
3163
3164 /* This function builds the iterators for a task & parameter */
3165
3166 void builditerators(struct taskdescriptor * task, int index, struct parameterwrapper * parameter) {
3167   int statusarray[MAXTASKPARAMS];
3168   int i;
3169   int numparams=task->numParameters;
3170   int iteratorcount=0;
3171   for(i=0;i<MAXTASKPARAMS;i++) statusarray[i]=0;
3172
3173   statusarray[index]=1; /* Initial parameter */
3174   /* Process tags for initial iterator */
3175
3176   processtags(task->descriptorarray[index], index, parameter, & iteratorcount, statusarray, numparams);
3177
3178   while(1) {
3179   loopstart:
3180     /* Check for objects with existing tags */
3181     for(i=0;i<numparams;i++) {
3182       if (statusarray[i]==0) {
3183         struct parameterdescriptor *pd=task->descriptorarray[i];
3184         int j;
3185         for(j=0;j<pd->numbertags;j++) {
3186           int slotid=pd->tagarray[2*j];
3187           if(statusarray[slotid+numparams]!=0) {
3188             processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3189             processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3190             goto loopstart;
3191           }
3192         }
3193       }
3194     }
3195
3196     /* Next do objects w/ unbound tags*/
3197
3198     for(i=0;i<numparams;i++) {
3199       if (statusarray[i]==0) {
3200         struct parameterdescriptor *pd=task->descriptorarray[i];
3201         if (pd->numbertags>0) {
3202           processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3203           processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3204           goto loopstart;
3205         }
3206       }
3207     }
3208
3209     /* Nothing with a tag enqueued */
3210
3211     for(i=0;i<numparams;i++) {
3212       if (statusarray[i]==0) {
3213         struct parameterdescriptor *pd=task->descriptorarray[i];
3214         processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3215         processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3216         goto loopstart;
3217       }
3218     }
3219
3220     /* Nothing left */
3221     return;
3222   }
3223 }
3224
3225  void printdebug() {
3226    int i;
3227    int j;
3228 #ifdef THREADSIMULATE
3229    int numofcore = pthread_getspecific(key);
3230    for(i=0;i<numtasks[numofcore];i++) {
3231            struct taskdescriptor * task=taskarray[numofcore][i];
3232 #else
3233 #ifdef RAW
3234            if(corenum > NUMCORES - 1) {
3235                    return;
3236            }
3237 #endif
3238    for(i=0;i<numtasks[corenum];i++) {
3239      struct taskdescriptor * task=taskarray[corenum][i];
3240 #endif
3241 #ifndef RAW
3242      printf("%s\n", task->name);
3243 #endif
3244      for(j=0;j<task->numParameters;j++) {
3245        struct parameterdescriptor *param=task->descriptorarray[j];
3246        struct parameterwrapper *parameter=param->queue;
3247        struct ObjectHash * set=parameter->objectset;
3248        struct ObjectIterator objit;
3249 #ifndef RAW
3250        printf("  Parameter %d\n", j);
3251 #endif
3252        ObjectHashiterator(set, &objit);
3253        while(ObjhasNext(&objit)) {
3254          struct ___Object___ * obj=(struct ___Object___ *)Objkey(&objit);
3255          struct ___Object___ * tagptr=obj->___tags___;
3256          int nonfailed=Objdata4(&objit);
3257          int numflags=Objdata3(&objit);
3258          int flags=Objdata2(&objit);
3259          Objnext(&objit);
3260 #ifndef RAW
3261          printf("    Contains %lx\n", obj);
3262          printf("      flag=%d\n", obj->flag);
3263 #endif
3264          if (tagptr==NULL) {
3265          } else if (tagptr->type==TAGTYPE) {
3266 #ifndef RAW
3267            printf("      tag=%lx\n",tagptr);
3268 #endif
3269            ;
3270          } else {
3271            int tagindex=0;
3272            struct ArrayObject *ao=(struct ArrayObject *)tagptr;
3273            for(;tagindex<ao->___cachedCode___;tagindex++) {
3274 #ifndef RAW
3275              printf("      tag=%lx\n",ARRAYGET(ao, struct ___TagDescriptor___*, tagindex));
3276 #endif
3277            }
3278          }
3279        }
3280      }
3281    }
3282  }
3283
3284
3285 /* This function processes the task information to create queues for
3286    each parameter type. */
3287
3288 void processtasks() {
3289   int i;
3290 #ifdef RAW
3291           if(corenum > NUMCORES - 1) {
3292                   return;
3293           }
3294 #endif
3295 #ifdef THREADSIMULATE
3296   int numofcore = pthread_getspecific(key);
3297   for(i=0;i<numtasks[numofcore];i++) {
3298           struct taskdescriptor *task=taskarray[numofcore][i];
3299 #else
3300   for(i=0;i<numtasks[corenum];i++) {
3301     struct taskdescriptor * task=taskarray[corenum][i];
3302 #endif
3303     int j;
3304
3305         /* Build objectsets */
3306         for(j=0;j<task->numParameters;j++) {
3307       struct parameterdescriptor *param=task->descriptorarray[j];
3308       struct parameterwrapper *parameter=param->queue;
3309           parameter->objectset=allocateObjectHash(10);
3310           parameter->task=task;
3311     }
3312
3313     /* Build iterators for parameters */
3314     for(j=0;j<task->numParameters;j++) {
3315       struct parameterdescriptor *param=task->descriptorarray[j];
3316       struct parameterwrapper *parameter=param->queue;
3317       builditerators(task, j, parameter);
3318     }
3319  }
3320 }
3321
3322 void toiReset(struct tagobjectiterator * it) {
3323   if (it->istag) {
3324     it->tagobjindex=0;
3325   } else if (it->numtags>0) {
3326     it->tagobjindex=0;
3327   } else {
3328     ObjectHashiterator(it->objectset, &it->it);
3329   }
3330 }
3331
3332 int toiHasNext(struct tagobjectiterator *it, void ** objectarray OPTARG(int * failed)) {
3333   if (it->istag) {
3334     /* Iterate tag */
3335     /* Get object with tags */
3336     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3337     struct ___Object___ *tagptr=obj->___tags___;
3338     if (tagptr->type==TAGTYPE) {
3339       if ((it->tagobjindex==0)&& /* First object */
3340           (it->tagid==((struct ___TagDescriptor___ *)tagptr)->flag)) /* Right tag type */
3341         return 1;
3342       else
3343         return 0;
3344     } else {
3345       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3346       int tagindex=it->tagobjindex;
3347       for(;tagindex<ao->___cachedCode___;tagindex++) {
3348         struct ___TagDescriptor___ *td=ARRAYGET(ao, struct ___TagDescriptor___ *, tagindex);
3349         if (td->flag==it->tagid) {
3350           it->tagobjindex=tagindex; /* Found right type of tag */
3351           return 1;
3352         }
3353       }
3354       return 0;
3355     }
3356   } else if (it->numtags>0) {
3357     /* Use tags to locate appropriate objects */
3358     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3359     struct ___Object___ *objptr=tag->flagptr;
3360     int i;
3361     if (objptr->type!=OBJECTARRAYTYPE) {
3362       if (it->tagobjindex>0)
3363         return 0;
3364       if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3365         return 0;
3366       for(i=1;i<it->numtags;i++) {
3367         struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3368         if (!containstag(objptr,tag2))
3369           return 0;
3370       }
3371       return 1;
3372     } else {
3373       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3374       int tagindex;
3375       int i;
3376       for(tagindex=it->tagobjindex;tagindex<ao->___cachedCode___;tagindex++) {
3377         struct ___Object___ *objptr=ARRAYGET(ao, struct ___Object___*, tagindex);
3378         if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3379           continue;
3380         for(i=1;i<it->numtags;i++) {
3381           struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3382           if (!containstag(objptr,tag2))
3383             goto nexttag;
3384         }
3385         it->tagobjindex=tagindex;
3386         return 1;
3387       nexttag:
3388         ;
3389       }
3390       it->tagobjindex=tagindex;
3391       return 0;
3392     }
3393   } else {
3394     return ObjhasNext(&it->it);
3395   }
3396 }
3397
3398 int containstag(struct ___Object___ *ptr, struct ___TagDescriptor___ *tag) {
3399   int j;
3400   struct ___Object___ * objptr=tag->flagptr;
3401   if (objptr->type==OBJECTARRAYTYPE) {
3402     struct ArrayObject *ao=(struct ArrayObject *)objptr;
3403     for(j=0;j<ao->___cachedCode___;j++) {
3404       if (ptr==ARRAYGET(ao, struct ___Object___*, j))
3405         return 1;
3406     }
3407     return 0;
3408   } else
3409     return objptr==ptr;
3410 }
3411
3412 void toiNext(struct tagobjectiterator *it , void ** objectarray OPTARG(int * failed)) {
3413   /* hasNext has all of the intelligence */
3414   if(it->istag) {
3415     /* Iterate tag */
3416     /* Get object with tags */
3417     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3418     struct ___Object___ *tagptr=obj->___tags___;
3419     if (tagptr->type==TAGTYPE) {
3420       it->tagobjindex++;
3421       objectarray[it->slot]=tagptr;
3422     } else {
3423       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3424       objectarray[it->slot]=ARRAYGET(ao, struct ___TagDescriptor___ *, it->tagobjindex++);
3425     }
3426   } else if (it->numtags>0) {
3427     /* Use tags to locate appropriate objects */
3428     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3429     struct ___Object___ *objptr=tag->flagptr;
3430     if (objptr->type!=OBJECTARRAYTYPE) {
3431       it->tagobjindex++;
3432       objectarray[it->slot]=objptr;
3433     } else {
3434       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3435       objectarray[it->slot]=ARRAYGET(ao, struct ___Object___ *, it->tagobjindex++);
3436     }
3437   } else {
3438     /* Iterate object */
3439     objectarray[it->slot]=(void *)Objkey(&it->it);
3440     Objnext(&it->it);
3441   }
3442 }
3443 #endif