Robust/src/Runtime/multicoretask.c

   1 #ifdef TASK
   2 #include "runtime.h"
   3 #ifndef RAW
   4 #include "structdefs.h"
   5 #include "mem.h"
   6 #include "checkpoint.h"
   7 #include "Queue.h"
   8 #include "SimpleHash.h"
   9 #include "GenericHashtable.h"
  10 #include <sys/select.h>
  11 #include <sys/types.h>
  12 #include <sys/mman.h>
  13 #include <string.h>
  14 #include <signal.h>
  15 #include <assert.h>
  16 #include <errno.h>
  17 #endif
  18 #ifdef RAW
  19 #include <raw.h>
  20 #include <raw_compiler_defs.h>
  21 //#include <libints.h>
  22 #elif defined THREADSIMULATE
  23 // use POSIX message queue
  24 // for each core, its message queue named as
  25 // /msgqueue_corenum
  26 #include <mqueue.h>
  27 #include <sys/stat.h>
  28 #endif
  29 /*
  30 extern int injectfailures;
  31 extern float failurechance;
  32 */
  33 extern int debugtask;
  34 extern int instaccum;
  35
  36 #ifdef RAW
  37 #define TOTALCORE raw_get_num_tiles()
  38 #endif
  39
  40 #ifdef CONSCHECK
  41 #include "instrument.h"
  42 #endif
  43
  44 struct genhashtable * activetasks;
  45 struct genhashtable * failedtasks;
  46 struct taskparamdescriptor * currtpd;
  47 #ifndef RAW
  48 struct RuntimeHash * forward;
  49 struct RuntimeHash * reverse;
  50 #endif
  51
  52 int corestatus[NUMCORES]; // records status of each core
  53                           // 1: running tasks
  54                                                   // 0: stall
  55 int numsendobjs[NUMCORES]; // records how many objects a core has sent out
  56 int numreceiveobjs[NUMCORES]; // records how many objects a core has received
  57 #ifdef RAW
  58 struct RuntimeHash locktable;
  59 static struct RuntimeHash* locktbl = &locktable;
  60 void * curr_heapbase=0;
  61 void * curr_heaptop=0;
  62 int self_numsendobjs;
  63 int self_numreceiveobjs;
  64 int lockobj;
  65 int lockresult;
  66 bool lockflag;
  67 #ifndef INTERRUPT
  68 bool reside;
  69 #endif
  70 struct Queue objqueue;
  71 int msgdata[30];
  72 int msgtype;
  73 int msgdataindex;
  74 int msglength;
  75 void calCoords(int core_num, int* coordY, int* coordX);
  76 #elif defined THREADSIMULATE
  77 static struct RuntimeHash* locktbl;
  78 struct thread_data {
  79         int corenum;
  80         int argc;
  81         char** argv;
  82         int numsendobjs;
  83         int numreceiveobjs;
  84 };
  85 struct thread_data thread_data_array[NUMCORES];
  86 mqd_t mqd[NUMCORES];
  87 static pthread_key_t key;
  88 static pthread_rwlock_t rwlock_tbl;
  89 static pthread_rwlock_t rwlock_init;
  90
  91 void run(void * arg);
  92 #endif
  93
  94 bool transStallMsg(int targetcore);
  95 void transTerminateMsg(int targetcore);
  96 int receiveObject();
  97 bool getreadlock(void* ptr);
  98 void releasereadlock(void* ptr);
  99 #ifdef RAW
 100 bool getreadlock_I(void* ptr);
 101 void releasereadlock_I(void* ptr);
 102 #endif
 103 bool getwritelock(void* ptr);
 104 void releasewritelock(void* ptr);
 105
 106 #ifdef RAW
 107 void begin() {
 108 #else
 109 int main(int argc, char **argv) {
 110 #endif
 111 #ifdef RAW
 112         int i = 0;
 113         int argc = 1;
 114         char ** argv = NULL;
 115         bool sendStall = false;
 116         bool isfirst = true;
 117         bool tocontinue = false;
 118         struct QueueItem * objitem = NULL;
 119         struct transObjInfo * objInfo = NULL;
 120         int grount = 0;
 121         bool allStall = true;
 122         int sumsendobj = 0;
 123
 124 #ifdef RAWDEBUG
 125         raw_test_pass(0xee01);
 126 #endif
 127         corenum = raw_get_abs_pos_x() + 4 * raw_get_abs_pos_y();
 128
 129         // initialize the arrays
 130         if(STARTUPCORE == corenum) {
 131                 // startup core to initialize corestatus[]
 132                 for(i = 0; i < NUMCORES; ++i) {
 133                         corestatus[i] = 1;
 134                         numsendobjs[i] = 0; // assume all variables in RAW are local variables! MAY BE WRONG!!!
 135                         numreceiveobjs[i] = 0;
 136                 }
 137         }
 138         self_numsendobjs = 0;
 139         self_numreceiveobjs = 0;
 140         for(i = 0; i < 30; ++i) {
 141                 msgdata[i] = -1;
 142         }
 143         //msgdata = NULL;
 144         msgtype = -1;
 145         msgdataindex = 0;
 146         msglength = 30;
 147 #ifdef RAWDEBUG
 148         raw_test_pass(0xee02);
 149 #endif
 150
 151         // create the lock table, lockresult table and obj queue
 152         locktable.size = 20;
 153     locktable.bucket = (struct RuntimeNode **) RUNMALLOC_I(sizeof(struct RuntimeNode *)*20);
 154     /* Set allocation blocks*/
 155     locktable.listhead=NULL;
 156     locktable.listtail=NULL;
 157     /*Set data counts*/
 158     locktable.numelements = 0;
 159         lockobj = 0;
 160         lockresult = 0;
 161         lockflag = false;
 162 #ifndef INTERRUPT
 163         reside = false;
 164 #endif
 165         objqueue.head = NULL;
 166         objqueue.tail = NULL;
 167 #ifdef RAWDEBUG
 168         raw_test_pass(0xee03);
 169 #endif
 170
 171 #ifdef INTERRUPT
 172         if (corenum < NUMCORES) {
 173         // set up interrupts
 174         setup_ints();
 175         //setup_interrupts();
 176         //start_gdn_avail_ints(recvMsg);
 177         raw_user_interrupts_on();
 178 #ifdef RAWDEBUG
 179         raw_test_pass(0xee04);
 180 #endif
 181         }
 182 #endif
 183
 184 #elif defined THREADSIMULATE
 185         errno = 0;
 186         int tids[NUMCORES];
 187         int rc[NUMCORES];
 188         pthread_t threads[NUMCORES];
 189         int i = 0;
 190
 191         // initialize three arrays and msg queue array
 192         char * pathhead = "/msgqueue_";
 193         int targetlen = strlen(pathhead);
 194         for(i = 0; i < NUMCORES; ++i) {
 195                 corestatus[i] = 1;
 196                 numsendobjs[i] = 0;
 197                 numreceiveobjs[i] = 0;
 198
 199                 char corenumstr[3];
 200                 int sourcelen = 0;
 201                 if(i < 10) {
 202                         corenumstr[0] = i + '0';
 203                         corenumstr[1] = '\0';
 204                         sourcelen = 1;
 205                 } else if(i < 100) {
 206                         corenumstr[1] = i %10 + '0';
 207                         corenumstr[0] = (i / 10) + '0';
 208                         corenumstr[2] = '\0';
 209                         sourcelen = 2;
 210                 } else {
 211                         printf("Error: i >= 100\n");
 212                         fflush(stdout);
 213                         exit(-1);
 214                 }
 215                 char path[targetlen + sourcelen + 1];
 216                 strcpy(path, pathhead);
 217                 strncat(path, corenumstr, sourcelen);
 218                 int oflags = O_RDONLY|O_CREAT|O_NONBLOCK;
 219                 int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
 220                 mq_unlink(path);
 221                 mqd[i]= mq_open(path, oflags, omodes, NULL);
 222                 if(mqd[i] == -1) {
 223                         printf("[Main] mq_open %s fails: %d, error: %s\n", path, mqd[i], strerror(errno));
 224                         exit(-1);
 225                 } else {
 226                         printf("[Main] mq_open %s returns: %d\n", path, mqd[i]);
 227                 }
 228         }
 229
 230         // create the key
 231         pthread_key_create(&key, NULL);
 232
 233         // create the lock table and initialize its mutex
 234         locktbl = allocateRuntimeHash(20);
 235         int rc_locktbl = pthread_rwlock_init(&rwlock_tbl, NULL);
 236         printf("[Main] initialize the rwlock for lock table: %d error: \n", rc_locktbl, strerror(rc_locktbl));
 237
 238         for(i = 0; i < NUMCORES; ++i) {
 239                 thread_data_array[i].corenum = i;
 240                 thread_data_array[i].argc = argc;
 241                 thread_data_array[i].argv = argv;
 242                 thread_data_array[i].numsendobjs = 0;
 243                 thread_data_array[i].numreceiveobjs = 0;
 244                 printf("[main] creating thread %d\n", i);
 245                 rc[i] = pthread_create(&threads[i], NULL, run, (void *)&thread_data_array[i]);
 246         if (rc[i]){
 247                         printf("[main] ERROR; return code from pthread_create() is %d\n", rc[i]);
 248                         fflush(stdout);
 249                         exit(-1);
 250                 }
 251         }
 252
 253         //pthread_exit(NULL);
 254         while(true) {}
 255 }
 256
 257 void run(void* arg) {
 258         struct thread_data * my_tdata = (struct thread_data *)arg;
 259         pthread_setspecific(key, (void *)my_tdata->corenum);
 260         int argc = my_tdata->argc;
 261         char** argv = my_tdata->argv;
 262         printf("[run, %d] Thread %d runs: %x\n", my_tdata->corenum, my_tdata->corenum, (int)pthread_self());
 263         fflush(stdout);
 264
 265 #endif
 266
 267 #ifdef BOEHM_GC
 268   GC_init(); // Initialize the garbage collector
 269 #endif
 270 #ifdef CONSCHECK
 271   initializemmap();
 272 #endif
 273 #ifndef RAW
 274   processOptions();
 275 #endif
 276   initializeexithandler();
 277 #ifdef RAWDEBUG
 278   raw_test_pass(0xee05);
 279 #endif
 280   /* Create table for failed tasks */
 281 #ifdef RAW
 282   if(corenum > NUMCORES - 1) {
 283           failedtasks = NULL;
 284           activetasks = NULL;
 285           while(true) {
 286                   receiveObject();
 287           }
 288   } else {
 289 #ifdef RAWDEBUG
 290           raw_test_pass(0xee06);
 291 #endif
 292 #endif
 293   /*failedtasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 294                                    (int (*)(void *,void *)) &comparetpd);*/
 295           failedtasks = NULL;
 296 #ifdef RAWDEBUG
 297   raw_test_pass(0xee07);
 298 #endif
 299   /* Create queue of active tasks */
 300   activetasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 301                                    (int (*)(void *,void *)) &comparetpd);
 302 #ifdef RAWDEBUG
 303   raw_test_pass(0xee08);
 304 #endif
 305
 306   /* Process task information */
 307   processtasks();
 308 #ifdef RAWDEBUG
 309   raw_test_pass(0xee09);
 310 #endif
 311
 312   if(STARTUPCORE == corenum) {
 313   /* Create startup object */
 314   createstartupobject(argc, argv);
 315   }
 316 #ifdef RAWDEBUG
 317   raw_test_pass(0xee0a);
 318 #endif
 319
 320 #ifdef RAW
 321 #ifdef RAWDEBUG
 322   raw_test_pass(0xee0b);
 323 #endif
 324
 325   while(true) {
 326 /*#ifndef INTERRUPT
 327           while(receiveObject() != -1) {
 328           }
 329 #endif*/
 330
 331           // check if there are new active tasks can be executed
 332           executetasks();
 333
 334 #ifndef INTERRUPT
 335           while(receiveObject() != -1) {
 336           }
 337 #endif
 338
 339 #ifdef RAWDEBUG
 340           raw_test_pass(0xee0c);
 341 #endif
 342
 343           // check if there are some pending objects, if yes, enqueue them and executetasks again
 344           tocontinue = false;
 345 #ifdef RAWDEBUG
 346           raw_test_pass(0xee0d);
 347 #endif
 348           while(!isEmpty(&objqueue)) {
 349                   void * obj = NULL;
 350 #ifdef INTERRUPT
 351                   raw_user_interrupts_off();
 352 #endif
 353 #ifdef RAWDEBUG
 354                   raw_test_pass(0xeee1);
 355 #endif
 356                   sendStall = false;
 357                   tocontinue = true;
 358                   objitem = getTail(&objqueue);
 359                   //obj = objitem->objectptr;
 360                   objInfo = (struct transObjInfo *)objitem->objectptr;
 361                   obj = objInfo->objptr;
 362 #ifdef RAWDEBUG
 363                   raw_test_pass_reg((int)obj);
 364 #endif
 365                   // grab lock and flush the obj
 366                   getreadlock_I(obj);
 367                   while(!lockflag) {
 368                           receiveObject();
 369                   }
 370                   grount = lockresult;
 371 #ifdef RAWDEBUG
 372                   raw_test_pass_reg(grount);
 373 #endif
 374
 375                   lockresult = 0;
 376                   lockobj = 0;
 377                   lockflag = false;
 378 #ifndef INTERRUPT
 379                   reside = false;
 380 #endif
 381
 382                   if(grount == 1) {
 383                           int k = 0;
 384                           raw_invalidate_cache_range(obj, classsize[((struct ___Object___ *)obj)->type]);
 385                           // flush the obj
 386                           /*for(k = 0; k < classsize[((struct ___Object___ *)obj)->type]; ++k) {
 387                                   invalidateAddr(obj + k);
 388                           }*/
 389                           // enqueue the object
 390                           for(k = 0; k < objInfo->length; ++k) {
 391                                   int taskindex = objInfo->queues[2 * k];
 392                                   int paramindex = objInfo->queues[2 * k + 1];
 393                                   struct parameterwrapper ** queues = &(paramqueues[corenum][taskindex][paramindex]);
 394 #ifdef RAWDEBUG
 395                                   raw_test_pass_reg(taskindex);
 396                                   raw_test_pass_reg(paramindex);
 397 #endif
 398                                   enqueueObject_I(obj, queues, 1);
 399                           }
 400                           removeItem(&objqueue, objitem);
 401                           releasereadlock_I(obj);
 402                           RUNFREE(objInfo->queues);
 403                           RUNFREE(objInfo);
 404                           /*enqueueObject_I(obj, NULL, 0);
 405                           removeItem(&objqueue, objitem);
 406                           releasereadlock_I(obj);*/
 407                   } else {
 408                           // can not get lock
 409                           // put it at the end of the queue
 410                           // and try to execute active tasks already enqueued first
 411                           removeItem(&objqueue, objitem);
 412                           addNewItem_I(&objqueue, objInfo);
 413                           break;
 414                   }
 415 #ifdef INTERRUPT
 416                   raw_user_interrupts_on();
 417 #endif
 418 #ifdef RAWDEBUG
 419                   raw_test_pass(0xee0e);
 420 #endif
 421           }
 422 #ifdef RAWDEBUG
 423           raw_test_pass(0xee0f);
 424 #endif
 425
 426           if(!tocontinue) {
 427                   // check if stop
 428                   if(STARTUPCORE == corenum) {
 429                           if(isfirst) {
 430 #ifdef RAWDEBUG
 431                                   raw_test_pass(0xee10);
 432 #endif
 433                                   isfirst = false;
 434                           }
 435 #ifdef INTERRUPT
 436                           raw_user_interrupts_off();
 437 #endif
 438                           corestatus[corenum] = 0;
 439                           numsendobjs[corenum] = self_numsendobjs;
 440                           numreceiveobjs[corenum] = self_numreceiveobjs;
 441                           // check the status of all cores
 442                           allStall = true;
 443 #ifdef RAWDEBUG
 444                           raw_test_pass_reg(NUMCORES);
 445 #endif
 446                           for(i = 0; i < NUMCORES; ++i) {
 447 #ifdef RAWDEBUG
 448                                   raw_test_pass(0xe000 + corestatus[i]);
 449 #endif
 450                                   if(corestatus[i] != 0) {
 451                                           allStall = false;
 452                                           break;
 453                                   }
 454                           }
 455                           if(allStall) {
 456                                   // check if the sum of send objs and receive obj are the same
 457                                   // yes->terminate
 458                                   // no->go on executing
 459                                   sumsendobj = 0;
 460                                   for(i = 0; i < NUMCORES; ++i) {
 461                                           sumsendobj += numsendobjs[i];
 462 #ifdef RAWDEBUG
 463                                           raw_test_pass(0xf000 + numsendobjs[i]);
 464 #endif
 465                                   }
 466                                   for(i = 0; i < NUMCORES; ++i) {
 467                                           sumsendobj -= numreceiveobjs[i];
 468 #ifdef RAWDEBUG
 469                                           raw_test_pass(0xf000 + numreceiveobjs[i]);
 470 #endif
 471                                   }
 472                                   if(0 == sumsendobj) {
 473                                           // terminate
 474 #ifdef RAWDEBUG
 475                                           raw_test_pass(0xee11);
 476 #endif
 477                                           raw_test_pass(raw_get_cycle());
 478                                           raw_test_done(1);     // All done.
 479                                   }
 480                           }
 481 #ifdef INTERRUPT
 482                           raw_user_interrupts_on();
 483 #endif
 484                   } else {
 485                           if(!sendStall) {
 486 #ifdef RAWDEBUG
 487                                   raw_test_pass(0xee12);
 488 #endif
 489                                   if(isfirst) {
 490                                           // wait for some time
 491                                           int halt = 10000;
 492 #ifdef RAWDEBUG
 493                                           raw_test_pass(0xee13);
 494 #endif
 495                                           while(halt--){}
 496                                           isfirst = false;
 497 #ifdef RAWDEBUG
 498                                           raw_test_pass(0xee14);
 499 #endif
 500                                   } else {
 501                                         // send StallMsg to startup core
 502 #ifdef RAWDEBUG
 503                                         raw_test_pass(0xee15);
 504 #endif
 505                                         sendStall = transStallMsg(STARTUPCORE);
 506                                         isfirst = true;
 507                                   }
 508                           } else {
 509                                   isfirst = true;
 510 #ifdef RAWDEBUG
 511                                   raw_test_pass(0xee16);
 512 #endif
 513                           }
 514                   }
 515           }
 516   }
 517   }
 518 #elif defined THREADSIMULATE
 519   /* Start executing the tasks */
 520   executetasks();
 521
 522   int i = 0;
 523   // check if there are new objects coming
 524   bool sendStall = false;
 525
 526   int numofcore = pthread_getspecific(key);
 527   while(true) {
 528           switch(receiveObject()) {
 529                   case 0: {
 530                                           printf("[run, %d] receive an object\n", numofcore);
 531                                           sendStall = false;
 532                                           // received an object
 533                                           // check if there are new active tasks can be executed
 534                                           executetasks();
 535                                           break;
 536                                   }
 537                   case 1: {
 538                                           //printf("[run, %d] no msg\n", numofcore);
 539                                           // no msg received
 540                                           if(STARTUPCORE == numofcore) {
 541                                                   corestatus[numofcore] = 0;
 542                                                   // check the status of all cores
 543                                                   bool allStall = true;
 544                                                   for(i = 0; i < NUMCORES; ++i) {
 545                                                           if(corestatus[i] != 0) {
 546                                                                   allStall = false;
 547                                                                   break;
 548                                                           }
 549                                                   }
 550                                                   if(allStall) {
 551                                                           // check if the sum of send objs and receive obj are the same
 552                                                           // yes->terminate
 553                                                           // no->go on executing
 554                                                           int sumsendobj = 0;
 555                                                           for(i = 0; i < NUMCORES; ++i) {
 556                                                                   sumsendobj += numsendobjs[i];
 557                                                           }
 558                                                           for(i = 0; i < NUMCORES; ++i) {
 559                                                                   sumsendobj -= numreceiveobjs[i];
 560                                                           }
 561                                                           if(0 == sumsendobj) {
 562                                                                   // terminate
 563
 564                                                                   // release all locks
 565                                                                   int rc_tbl = pthread_rwlock_wrlock(&rwlock_tbl);
 566                                                                   printf("[run, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc_tbl, strerror(rc_tbl));
 567                                                                   struct RuntimeIterator* it_lock = RuntimeHashcreateiterator(locktbl);
 568                                                                   while(0 != RunhasNext(it_lock)) {
 569                                                                           int key = Runkey(it_lock);
 570                                                                           pthread_rwlock_t* rwlock_obj = (pthread_rwlock_t*)Runnext(it_lock);
 571                                                                           int rc_des = pthread_rwlock_destroy(rwlock_obj);
 572                                                                           printf("[run, %d] destroy the rwlock for object: %d error: \n", numofcore, key, strerror(rc_des));
 573                                                                           RUNFREE(rwlock_obj);
 574                                                                   }
 575                                                                   freeRuntimeHash(locktbl);
 576                                                                   locktbl = NULL;
 577                                                                   RUNFREE(it_lock);
 578
 579                                                                   // destroy all message queues
 580                                                                   char * pathhead = "/msgqueue_";
 581                                                                   int targetlen = strlen(pathhead);
 582                                                                   for(i = 0; i < NUMCORES; ++i) {
 583                                                                           char corenumstr[3];
 584                                                                           int sourcelen = 0;
 585                                                                           if(i < 10) {
 586                                                                                   corenumstr[0] = i + '0';
 587                                                                                   corenumstr[1] = '\0';
 588                                                                                   sourcelen = 1;
 589                                                                           } else if(i < 100) {
 590                                                                                   corenumstr[1] = i %10 + '0';
 591                                                                                   corenumstr[0] = (i / 10) + '0';
 592                                                                                   corenumstr[2] = '\0';
 593                                                                                   sourcelen = 2;
 594                                                                           } else {
 595                                                                                   printf("Error: i >= 100\n");
 596                                                                                   fflush(stdout);
 597                                                                                   exit(-1);
 598                                                                           }
 599                                                                           char path[targetlen + sourcelen + 1];
 600                                                                           strcpy(path, pathhead);
 601                                                                           strncat(path, corenumstr, sourcelen);
 602                                                                           mq_unlink(path);
 603                                                                   }
 604
 605                                                                   printf("[run, %d] terminate!\n", numofcore);
 606                                                                   fflush(stdout);
 607                                                                   exit(0);
 608                                                           }
 609                                                   }
 610                                           } else {
 611                                                   if(!sendStall) {
 612                                                           // send StallMsg to startup core
 613                                                           sendStall = transStallMsg(STARTUPCORE);
 614                                                   }
 615                                           }
 616                                           break;
 617                                   }
 618                   case 2: {
 619                                           printf("[run, %d] receive a stall msg\n", numofcore);
 620                                           // receive a Stall Msg, do nothing
 621                                           assert(STARTUPCORE == numofcore); // only startup core can receive such msg
 622                                           sendStall = false;
 623                                           break;
 624                                   }
 625                  /* case 3: {
 626                                           printf("[run, %d] receive a terminate msg\n", numofcore);
 627                                           // receive a terminate Msg
 628                                           assert(STARTUPCORE != corenum); // only non-startup core can receive such msg
 629                                           mq_close(mqd[corenum]);
 630                                           fflush(stdout);
 631                                           exit(0);
 632                                           break;
 633                                   }*/
 634                   default: {
 635                                            printf("[run, %d] Error: invalid message type.\n", numofcore);
 636                                            fflush(stdout);
 637                                            exit(-1);
 638                                            break;
 639                                    }
 640           }
 641   }
 642 #endif
 643 }
 644
 645 void createstartupobject(int argc, char ** argv) {
 646   int i;
 647
 648   /* Allocate startup object     */
 649 #ifdef PRECISE_GC
 650   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(NULL, STARTUPTYPE);
 651   struct ArrayObject * stringarray=allocate_newarray(NULL, STRINGARRAYTYPE, argc-1);
 652 #else
 653   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(STARTUPTYPE);
 654   struct ArrayObject * stringarray=allocate_newarray(STRINGARRAYTYPE, argc-1);
 655 #endif
 656   /* Build array of strings */
 657   startupobject->___parameters___=stringarray;
 658   for(i=1;i<argc;i++) {
 659     int length=strlen(argv[i]);
 660 #ifdef PRECISE_GC
 661     struct ___String___ *newstring=NewString(NULL, argv[i],length);
 662 #else
 663     struct ___String___ *newstring=NewString(argv[i],length);
 664 #endif
 665     ((void **)(((char *)& stringarray->___length___)+sizeof(int)))[i-1]=newstring;
 666   }
 667
 668   startupobject->isolate = 1;
 669   startupobject->version = 0;
 670
 671   /* Set initialized flag for startup object */
 672   flagorandinit(startupobject,1,0xFFFFFFFF);
 673   enqueueObject(startupobject, NULL, 0);
 674 #ifdef RAW
 675   //flushAll();
 676   raw_flush_entire_cache();
 677 #endif
 678 }
 679
 680 int hashCodetpd(struct taskparamdescriptor *ftd) {
 681   int hash=(int)ftd->task;
 682   int i;
 683   for(i=0;i<ftd->numParameters;i++){
 684     hash^=(int)ftd->parameterArray[i];
 685   }
 686   return hash;
 687 }
 688
 689 int comparetpd(struct taskparamdescriptor *ftd1, struct taskparamdescriptor *ftd2) {
 690   int i;
 691   if (ftd1->task!=ftd2->task)
 692     return 0;
 693   for(i=0;i<ftd1->numParameters;i++)
 694     if(ftd1->parameterArray[i]!=ftd2->parameterArray[i])
 695       return 0;
 696   return 1;
 697 }
 698
 699 /* This function sets a tag. */
 700 #ifdef PRECISE_GC
 701 void tagset(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 702 #else
 703 void tagset(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 704 #endif
 705         struct ArrayObject * ao=NULL;
 706   struct ___Object___ * tagptr=obj->___tags___;
 707 #ifdef RAWDEBUG
 708   raw_test_pass(0xebb0);
 709 #endif
 710   if (tagptr==NULL) {
 711 #ifdef RAWDEBUG
 712           raw_test_pass(0xebb1);
 713 #endif
 714     obj->___tags___=(struct ___Object___ *)tagd;
 715   } else {
 716     /* Have to check if it is already set */
 717     if (tagptr->type==TAGTYPE) {
 718       struct ___TagDescriptor___ * td=(struct ___TagDescriptor___ *) tagptr;
 719 #ifdef RAWDEBUG
 720           raw_test_pass(0xebb2);
 721 #endif
 722       if (td==tagd) {
 723 #ifdef RAWDEBUG
 724                   raw_test_pass(0xebb3);
 725 #endif
 726         return;
 727           }
 728 #ifdef PRECISE_GC
 729       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 730       struct ArrayObject * ao=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL);
 731       obj=(struct ___Object___ *)ptrarray[2];
 732       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 733       td=(struct ___TagDescriptor___ *) obj->___tags___;
 734 #else
 735 #ifdef RAWDEBUG
 736           raw_test_pass(0xebb4);
 737 #endif
 738       ao=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL);
 739 #endif
 740 #ifdef RAWDEBUG
 741           raw_test_pass(0xebb5);
 742 #endif
 743       ARRAYSET(ao, struct ___TagDescriptor___ *, 0, td);
 744       ARRAYSET(ao, struct ___TagDescriptor___ *, 1, tagd);
 745       obj->___tags___=(struct ___Object___ *) ao;
 746       ao->___cachedCode___=2;
 747 #ifdef RAWDEBUG
 748           raw_test_pass(0xebb6);
 749 #endif
 750     } else {
 751       /* Array Case */
 752       int i;
 753       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 754 #ifdef RAWDEBUG
 755           raw_test_pass(0xebb7);
 756 #endif
 757       for(i=0;i<ao->___cachedCode___;i++) {
 758         struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___*, i);
 759 #ifdef RAWDEBUG
 760                 raw_test_pass(0xebb8);
 761 #endif
 762         if (td==tagd) {
 763 #ifdef RAWDEBUG
 764                 raw_test_pass(0xebb9);
 765 #endif
 766           return;
 767         }
 768       }
 769       if (ao->___cachedCode___<ao->___length___) {
 770 #ifdef RAWDEBUG
 771                 raw_test_pass(0xebba);
 772 #endif
 773         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, tagd);
 774         ao->___cachedCode___++;
 775 #ifdef RAWDEBUG
 776         raw_test_pass(0xebbb);
 777 #endif
 778       } else {
 779 #ifdef PRECISE_GC
 780         int ptrarray[]={2,(int) ptr, (int) obj, (int) tagd};
 781         struct ArrayObject * aonew=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 782         obj=(struct ___Object___ *)ptrarray[2];
 783         tagd=(struct ___TagDescriptor___ *) ptrarray[3];
 784         ao=(struct ArrayObject *)obj->___tags___;
 785 #else
 786         struct ArrayObject * aonew=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 787 #endif
 788 #ifdef RAWDEBUG
 789                 raw_test_pass(0xebbc);
 790 #endif
 791         aonew->___cachedCode___=ao->___length___+1;
 792         for(i=0;i<ao->___length___;i++) {
 793 #ifdef RAWDEBUG
 794                 raw_test_pass(0xebbd);
 795 #endif
 796           ARRAYSET(aonew, struct ___TagDescriptor___*, i, ARRAYGET(ao, struct ___TagDescriptor___*, i));
 797         }
 798 #ifdef RAWDEBUG
 799                 raw_test_pass(0xebbe);
 800 #endif
 801         ARRAYSET(aonew, struct ___TagDescriptor___ *, ao->___length___, tagd);
 802 #ifdef RAWDEBUG
 803         raw_test_pass(0xebbf);
 804 #endif
 805       }
 806     }
 807   }
 808
 809   {
 810     struct ___Object___ * tagset=tagd->flagptr;
 811 #ifdef RAWDEBUG
 812         raw_test_pass(0xb008);
 813 #endif
 814     if(tagset==NULL) {
 815 #ifdef RAWDEBUG
 816                 raw_test_pass(0xb009);
 817 #endif
 818       tagd->flagptr=obj;
 819     } else if (tagset->type!=OBJECTARRAYTYPE) {
 820 #ifdef PRECISE_GC
 821       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 822       struct ArrayObject * ao=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 823       obj=(struct ___Object___ *)ptrarray[2];
 824       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 825 #else
 826       struct ArrayObject * ao=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 827 #endif
 828       ARRAYSET(ao, struct ___Object___ *, 0, tagd->flagptr);
 829       ARRAYSET(ao, struct ___Object___ *, 1, obj);
 830       ao->___cachedCode___=2;
 831       tagd->flagptr=(struct ___Object___ *)ao;
 832 #ifdef RAWDEBUG
 833           raw_test_pass(0xb00a);
 834 #endif
 835     } else {
 836       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 837       if (ao->___cachedCode___<ao->___length___) {
 838 #ifdef RAWDEBUG
 839                   raw_test_pass(0xb00b);
 840 #endif
 841         ARRAYSET(ao, struct ___Object___*, ao->___cachedCode___++, obj);
 842       } else {
 843         int i;
 844 #ifdef PRECISE_GC
 845         int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 846         struct ArrayObject * aonew=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL+ao->___length___);
 847         obj=(struct ___Object___ *)ptrarray[2];
 848         tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 849         ao=(struct ArrayObject *)tagd->flagptr;
 850 #else
 851         struct ArrayObject * aonew=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 852 #endif
 853         aonew->___cachedCode___=ao->___cachedCode___+1;
 854         for(i=0;i<ao->___length___;i++) {
 855           ARRAYSET(aonew, struct ___Object___*, i, ARRAYGET(ao, struct ___Object___*, i));
 856         }
 857         ARRAYSET(aonew, struct ___Object___ *, ao->___cachedCode___, obj);
 858         tagd->flagptr=(struct ___Object___ *) aonew;
 859 #ifdef RAWDEBUG
 860         raw_test_pass(0xb00c);
 861 #endif
 862       }
 863     }
 864   }
 865 }
 866
 867 /* This function clears a tag. */
 868 #ifdef PRECISE_GC
 869 void tagclear(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 870 #else
 871 void tagclear(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 872 #endif
 873   /* We'll assume that tag is alway there.
 874      Need to statically check for this of course. */
 875   struct ___Object___ * tagptr=obj->___tags___;
 876
 877   if (tagptr->type==TAGTYPE) {
 878     if ((struct ___TagDescriptor___ *)tagptr==tagd)
 879       obj->___tags___=NULL;
 880     else
 881 #ifndef RAW
 882       printf("ERROR 1 in tagclear\n");
 883 #endif
 884         ;
 885   } else {
 886     struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 887     int i;
 888     for(i=0;i<ao->___cachedCode___;i++) {
 889       struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___ *, i);
 890       if (td==tagd) {
 891         ao->___cachedCode___--;
 892         if (i<ao->___cachedCode___)
 893           ARRAYSET(ao, struct ___TagDescriptor___ *, i, ARRAYGET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___));
 894         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, NULL);
 895         if (ao->___cachedCode___==0)
 896           obj->___tags___=NULL;
 897         goto PROCESSCLEAR;
 898       }
 899     }
 900 #ifndef RAW
 901     printf("ERROR 2 in tagclear\n");
 902 #endif
 903         ;
 904   }
 905  PROCESSCLEAR:
 906   {
 907     struct ___Object___ *tagset=tagd->flagptr;
 908     if (tagset->type!=OBJECTARRAYTYPE) {
 909       if (tagset==obj)
 910         tagd->flagptr=NULL;
 911       else
 912 #ifndef RAW
 913         printf("ERROR 3 in tagclear\n");
 914 #endif
 915           ;
 916     } else {
 917       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 918       int i;
 919       for(i=0;i<ao->___cachedCode___;i++) {
 920         struct ___Object___ * tobj=ARRAYGET(ao, struct ___Object___ *, i);
 921         if (tobj==obj) {
 922           ao->___cachedCode___--;
 923           if (i<ao->___cachedCode___)
 924             ARRAYSET(ao, struct ___Object___ *, i, ARRAYGET(ao, struct ___Object___ *, ao->___cachedCode___));
 925           ARRAYSET(ao, struct ___Object___ *, ao->___cachedCode___, NULL);
 926           if (ao->___cachedCode___==0)
 927             tagd->flagptr=NULL;
 928           goto ENDCLEAR;
 929         }
 930       }
 931 #ifndef RAW
 932       printf("ERROR 4 in tagclear\n");
 933 #endif
 934     }
 935   }
 936  ENDCLEAR:
 937   return;
 938 }
 939
 940 /* This function allocates a new tag. */
 941 #ifdef PRECISE_GC
 942 struct ___TagDescriptor___ * allocate_tag(void *ptr, int index) {
 943   struct ___TagDescriptor___ * v=(struct ___TagDescriptor___ *) mygcmalloc((struct garbagelist *) ptr, classsize[TAGTYPE]);
 944 #else
 945 struct ___TagDescriptor___ * allocate_tag(int index) {
 946   struct ___TagDescriptor___ * v=FREEMALLOC(classsize[TAGTYPE]);
 947 #endif
 948   v->type=TAGTYPE;
 949   v->flag=index;
 950   return v;
 951 }
 952
 953
 954
 955 /* This function updates the flag for object ptr.  It or's the flag
 956    with the or mask and and's it with the andmask. */
 957
 958 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** queues, int length, bool isnew);
 959
 960  int flagcomp(const int *val1, const int *val2) {
 961    return (*val1)-(*val2);
 962  }
 963
 964 void flagorand(void * ptr, int ormask, int andmask, struct parameterwrapper ** queues, int length) {
 965     {
 966       int oldflag=((int *)ptr)[1];
 967       int flag=ormask|oldflag;
 968       flag&=andmask;
 969 #ifdef RAWDEBUG
 970           raw_test_pass_reg((int)ptr);
 971           raw_test_pass(0xaa000000 + oldflag);
 972           raw_test_pass(0xaa000000 + flag);
 973 #endif
 974           flagbody(ptr, flag, queues, length, false);
 975     }
 976 }
 977
 978 bool intflagorand(void * ptr, int ormask, int andmask) {
 979     {
 980       int oldflag=((int *)ptr)[1];
 981       int flag=ormask|oldflag;
 982       flag&=andmask;
 983       if (flag==oldflag) /* Don't do anything */
 984         return false;
 985       else {
 986                  flagbody(ptr, flag, NULL, 0, false);
 987                  return true;
 988           }
 989     }
 990 }
 991
 992 void flagorandinit(void * ptr, int ormask, int andmask) {
 993   int oldflag=((int *)ptr)[1];
 994   int flag=ormask|oldflag;
 995   flag&=andmask;
 996 #ifdef RAWDEBUG
 997           raw_test_pass(0xaa100000 + oldflag);
 998           raw_test_pass(0xaa100000 + flag);
 999 #endif
1000   flagbody(ptr,flag,NULL,0,true);
1001 }
1002
1003 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** vqueues, int vlength, bool isnew) {
1004   struct parameterwrapper * flagptr = NULL;
1005   int i = 0;
1006   struct parameterwrapper ** queues = vqueues;
1007   int length = vlength;
1008   int next;
1009   int UNUSED, UNUSED2;
1010   int * enterflags = NULL;
1011   if((!isnew) && (queues == NULL)) {
1012 #ifdef THREADSIMULATE
1013           int numofcore = pthread_getspecific(key);
1014           queues = objectqueues[numofcore][ptr->type];
1015           length = numqueues[numofcore][ptr->type];
1016 #else
1017 #ifdef RAW
1018           if(corenum < NUMCORES) {
1019 #endif
1020           queues = objectqueues[corenum][ptr->type];
1021           length = numqueues[corenum][ptr->type];
1022 #ifdef RAW
1023           } else {
1024                   return;
1025           }
1026 #endif
1027 #endif
1028   }
1029   ptr->flag=flag;
1030 #ifdef RAWDEBUG
1031   raw_test_pass(0xbb000000 + ptr->flag);
1032 #endif
1033
1034   /*Remove object from all queues */
1035   for(i = 0; i < length; ++i) {
1036           flagptr = queues[i];
1037     ObjectHashget(flagptr->objectset, (int) ptr, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
1038     ObjectHashremove(flagptr->objectset, (int)ptr);
1039     if (enterflags!=NULL)
1040       RUNFREE(enterflags);
1041   }
1042  }
1043
1044  void enqueueObject(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1045    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1046
1047   {
1048     struct QueueItem *tmpptr;
1049         struct parameterwrapper * parameter=NULL;
1050         int j;
1051         int i;
1052     struct parameterwrapper * prevptr=NULL;
1053     struct ___Object___ *tagptr=NULL;
1054         struct parameterwrapper ** queues = vqueues;
1055         int length = vlength;
1056 #ifdef RAW
1057         if(corenum > NUMCORES - 1) {
1058                 return;
1059         }
1060 #endif
1061         if(queues == NULL) {
1062 #ifdef THREADSIMULATE
1063                 int numofcore = pthread_getspecific(key);
1064                 queues = objectqueues[numofcore][ptr->type];
1065                 length = numqueues[numofcore][ptr->type];
1066 #else
1067                 queues = objectqueues[corenum][ptr->type];
1068                 length = numqueues[corenum][ptr->type];
1069 #endif
1070         }
1071     tagptr=ptr->___tags___;
1072
1073     /* Outer loop iterates through all parameter queues an object of
1074        this type could be in.  */
1075     for(j = 0; j < length; ++j) {
1076                 parameter = queues[j];
1077       /* Check tags */
1078       if (parameter->numbertags>0) {
1079         if (tagptr==NULL)
1080           goto nextloop;//that means the object has no tag but that param needs tag
1081         else if(tagptr->type==TAGTYPE) {//one tag
1082           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1083           for(i=0;i<parameter->numbertags;i++) {
1084             //slotid is parameter->tagarray[2*i];
1085             int tagid=parameter->tagarray[2*i+1];
1086             if (tagid!=tagptr->flag)
1087               goto nextloop; /*We don't have this tag */
1088            }
1089         } else {//multiple tags
1090           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1091           for(i=0;i<parameter->numbertags;i++) {
1092             //slotid is parameter->tagarray[2*i];
1093             int tagid=parameter->tagarray[2*i+1];
1094             int j;
1095             for(j=0;j<ao->___cachedCode___;j++) {
1096               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag)
1097                 goto foundtag;
1098             }
1099             goto nextloop;
1100           foundtag:
1101             ;
1102           }
1103         }
1104       }
1105
1106       /* Check flags */
1107       for(i=0;i<parameter->numberofterms;i++) {
1108         int andmask=parameter->intarray[i*2];
1109         int checkmask=parameter->intarray[i*2+1];
1110         if ((ptr->flag&andmask)==checkmask) {
1111 #ifdef RAWDEBUG
1112                 raw_test_pass(0xcc000000 + andmask);
1113                 raw_test_pass_reg((int)ptr);
1114                 raw_test_pass(0xcc000000 + ptr->flag);
1115                 raw_test_pass(0xcc000000 + checkmask);
1116 #endif
1117           enqueuetasks(parameter, prevptr, ptr, NULL, 0);
1118           prevptr=parameter;
1119           break;
1120         }
1121       }
1122     nextloop:
1123           ;
1124     }
1125   }
1126 }
1127
1128 #ifdef RAW
1129 void enqueueObject_I(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1130    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1131
1132   {
1133     struct QueueItem *tmpptr;
1134         struct parameterwrapper * parameter=NULL;
1135         int j;
1136         int i;
1137     struct parameterwrapper * prevptr=NULL;
1138     struct ___Object___ *tagptr=NULL;
1139         struct parameterwrapper ** queues = vqueues;
1140         int length = vlength;
1141 #ifdef RAW
1142         if(corenum > NUMCORES - 1) {
1143                 return;
1144         }
1145 #endif
1146         if(queues == NULL) {
1147 #ifdef THREADSIMULATE
1148                 int numofcore = pthread_getspecific(key);
1149                 queues = objectqueues[numofcore][ptr->type];
1150                 length = numqueues[numofcore][ptr->type];
1151 #else
1152                 queues = objectqueues[corenum][ptr->type];
1153                 length = numqueues[corenum][ptr->type];
1154 #endif
1155         }
1156 #ifdef RAWDEBUG
1157         raw_test_pass(0xeaa1);
1158         raw_test_pass_reg(queues);
1159         raw_test_pass_reg(length);
1160 #endif
1161     tagptr=ptr->___tags___;
1162
1163     /* Outer loop iterates through all parameter queues an object of
1164        this type could be in.  */
1165     for(j = 0; j < length; ++j) {
1166                 parameter = queues[j];
1167       /* Check tags */
1168       if (parameter->numbertags>0) {
1169 #ifdef RAWDEBUG
1170         raw_test_pass(0xeaa2);
1171         raw_test_pass_reg(tagptr);
1172 #endif
1173         if (tagptr==NULL)
1174           goto nextloop;//that means the object has no tag but that param needs tag
1175         else if(tagptr->type==TAGTYPE) {//one tag
1176           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1177 #ifdef RAWDEBUG
1178         raw_test_pass(0xeaa3);
1179 #endif
1180           for(i=0;i<parameter->numbertags;i++) {
1181             //slotid is parameter->tagarray[2*i];
1182             int tagid=parameter->tagarray[2*i+1];
1183             if (tagid!=tagptr->flag) {
1184 #ifdef RAWDEBUG
1185         raw_test_pass(0xeaa4);
1186 #endif
1187               goto nextloop; /*We don't have this tag */
1188                 }
1189            }
1190         } else {//multiple tags
1191           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1192 #ifdef RAWDEBUG
1193         raw_test_pass(0xeaa5);
1194 #endif
1195           for(i=0;i<parameter->numbertags;i++) {
1196             //slotid is parameter->tagarray[2*i];
1197             int tagid=parameter->tagarray[2*i+1];
1198             int j;
1199             for(j=0;j<ao->___cachedCode___;j++) {
1200               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag) {
1201                 goto foundtag;
1202                   }
1203             }
1204 #ifdef RAWDEBUG
1205         raw_test_pass(0xeaa6);
1206 #endif
1207             goto nextloop;
1208           foundtag:
1209             ;
1210           }
1211         }
1212       }
1213
1214       /* Check flags */
1215       for(i=0;i<parameter->numberofterms;i++) {
1216         int andmask=parameter->intarray[i*2];
1217         int checkmask=parameter->intarray[i*2+1];
1218 #ifdef RAWDEBUG
1219         raw_test_pass(0xeaa7);
1220         raw_test_pass(0xcc000000 + andmask);
1221         raw_test_pass_reg(ptr);
1222         raw_test_pass(0xcc000000 + ptr->flag);
1223         raw_test_pass(0xcc000000 + checkmask);
1224 #endif
1225         if ((ptr->flag&andmask)==checkmask) {
1226 #ifdef RAWDEBUG
1227         raw_test_pass(0xeaa8);
1228 #endif
1229           enqueuetasks_I(parameter, prevptr, ptr, NULL, 0);
1230           prevptr=parameter;
1231           break;
1232         }
1233       }
1234     nextloop:
1235           ;
1236     }
1237   }
1238 }
1239
1240 // helper function to compute the coordinates of a core from the core number
1241 void calCoords(int core_num, int* coordY, int* coordX) {
1242         *coordX = core_num % 4;
1243         *coordY = core_num / 4;
1244 }
1245 #endif
1246
1247 /* Message format for RAW version:
1248  *      type + Msgbody
1249  * type: 0 -- transfer object
1250  *       1 -- transfer stall msg
1251  *       2 -- lock request
1252  *       3 -- lock grount
1253  *       4 -- lock deny
1254  *       5 -- lock release
1255  *
1256  * ObjMsg: 0 + size of msg + obj's address + (task index + param index)+
1257  * StallMsg: 1 + corenum + sendobjs + receiveobjs (size is always 4 * sizeof(int))
1258  * LockMsg: 2 + lock type + obj pointer + request core (size is always 4 * sizeof(int))
1259  *          3/4/5 + lock type + obj pointer (size is always 3 * sizeof(int))
1260  *          lock type: 0 -- read; 1 -- write
1261  */
1262
1263 // transfer an object to targetcore
1264 // format: object
1265 void transferObject(struct transObjInfo * transObj) {
1266         void * obj = transObj->objptr;
1267         int type=((int *)obj)[0];
1268     int size=classsize[type];
1269         int targetcore = transObj->targetcore;
1270         //assert(type < NUMCLASSES); // can only transfer normal object
1271
1272 #ifdef RAW
1273         unsigned msgHdr;
1274         int self_y, self_x, target_y, target_x;
1275         //int isshared = 0;
1276         // for 32 bit machine, the size of fixed part is always 3 words
1277         //int msgsize = sizeof(int) * 2 + sizeof(void *);
1278         int msgsize = 3 + transObj->length * 2;
1279         int i = 0;
1280
1281         struct ___Object___ * newobj = (struct ___Object___ *)obj;
1282         /*if(0 == newobj->isolate) {
1283                 isshared = 1;
1284         }*/
1285
1286         calCoords(corenum, &self_y, &self_x);
1287         calCoords(targetcore, &target_y, &target_x);
1288         // Build the message header
1289     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1290                                            self_y, self_x,
1291                                                            target_y, target_x);
1292         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1293 #ifdef RAWDEBUG
1294         raw_test_pass(0xbbbb);
1295         raw_test_pass(0xb000 + targetcore); // targetcore
1296 #endif
1297     gdn_send(0);
1298 #ifdef RAWDEBUG
1299     raw_test_pass(0);
1300 #endif
1301         gdn_send(msgsize);
1302 #ifdef RAWDEBUG
1303         raw_test_pass_reg(msgsize);
1304 #endif
1305     gdn_send(obj);
1306 #ifdef RAWDEBUG
1307         raw_test_pass_reg(obj);
1308 #endif
1309         for(i = 0; i < transObj->length; ++i) {
1310                 int taskindex = transObj->queues[2*i];
1311                 int paramindex = transObj->queues[2*i+1];
1312                 gdn_send(taskindex);
1313 #ifdef RAWDEBUG
1314                 raw_test_pass_reg(taskindex);
1315 #endif
1316                 gdn_send(paramindex);
1317 #ifdef RAWDEBUG
1318                 raw_test_pass_reg(paramindex);
1319 #endif
1320         }
1321 #ifdef RAWDEBUG
1322         raw_test_pass(0xffff);
1323 #endif
1324         ++(self_numsendobjs);
1325 #elif defined THREADSIMULATE
1326         int numofcore = pthread_getspecific(key);
1327
1328         // use POSIX message queue to transfer objects between cores
1329         mqd_t mqdnum;
1330         char corenumstr[3];
1331         int sourcelen = 0;
1332         if(targetcore < 10) {
1333                 corenumstr[0] = targetcore + '0';
1334                 corenumstr[1] = '\0';
1335                 sourcelen = 1;
1336         } else if(targetcore < 100) {
1337                 corenumstr[1] = targetcore % 10 + '0';
1338                 corenumstr[0] = (targetcore / 10) + '0';
1339                 corenumstr[2] = '\0';
1340                 sourcelen = 2;
1341         } else {
1342                 printf("Error: targetcore >= 100\n");
1343                 fflush(stdout);
1344                 exit(-1);
1345         }
1346         char * pathhead = "/msgqueue_";
1347         int targetlen = strlen(pathhead);
1348         char path[targetlen + sourcelen + 1];
1349         strcpy(path, pathhead);
1350         strncat(path, corenumstr, sourcelen);
1351         int oflags = O_WRONLY|O_NONBLOCK;
1352         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1353         mqdnum = mq_open(path, oflags, omodes, NULL);
1354         if(mqdnum==-1) {
1355                 printf("[transferObject, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1356                 fflush(stdout);
1357                 exit(-1);
1358         }
1359         /*struct ___Object___ * newobj = (struct ___Object___ *)obj;
1360         if(0 == newobj->isolate) {
1361                 newobj = RUNMALLOC(size);
1362                 memcpy(newobj, obj, size);
1363                 newobj->original=obj;
1364         }*/
1365         struct transObjInfo * tmptransObj = RUNMALLOC(sizeof(struct transObjInfo));
1366         memcpy(tmptransObj, transObj, sizeof(struct transObjInfo));
1367         int * tmpqueue = RUNMALLOC(sizeof(int)*2*tmptransObj->length);
1368         memcpy(tmpqueue, tmptransObj->queues, sizeof(int)*2*tmptransObj->length);
1369         tmptransObj->queues = tmpqueue;
1370         struct ___Object___ * newobj = RUNMALLOC(sizeof(struct ___Object___));
1371         newobj->type = ((struct ___Object___ *)obj)->type;
1372         newobj->original = (struct ___Object___ *)tmptransObj;
1373         int ret;
1374         do {
1375                 ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1376                 if(ret != 0) {
1377                         printf("[transferObject, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1378                 }
1379         }while(ret!=0);
1380         RUNFREE(newobj);
1381         if(numofcore == STARTUPCORE) {
1382                 ++numsendobjs[numofcore];
1383         } else {
1384                 ++(thread_data_array[numofcore].numsendobjs);
1385         }
1386         printf("[transferObject, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1387 #endif
1388 }
1389
1390 // send terminate message to targetcore
1391 // format: -1
1392 bool transStallMsg(int targetcore) {
1393 #ifdef RAW
1394         unsigned msgHdr;
1395         int self_y, self_x, target_y, target_x;
1396         // for 32 bit machine, the size is always 4 words
1397         //int msgsize = sizeof(int) * 4;
1398         int msgsize = 4;
1399
1400         calCoords(corenum, &self_y, &self_x);
1401         calCoords(targetcore, &target_y, &target_x);
1402         // Build the message header
1403     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1404                                            self_y, self_x,
1405                                                            target_y, target_x);
1406         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1407 #ifdef RAWDEBUG
1408         raw_test_pass(0xbbbb);
1409         raw_test_pass(0xb000 + targetcore); // targetcore
1410 #endif
1411     gdn_send(1);
1412 #ifdef RAWDEBUG
1413     raw_test_pass(1);
1414 #endif
1415     gdn_send(corenum);
1416 #ifdef RAWDEBUG
1417         raw_test_pass_reg(corenum);
1418 #endif
1419         gdn_send(self_numsendobjs);
1420 #ifdef RAWDEBUG
1421     raw_test_pass_reg(self_numsendobjs);
1422 #endif
1423     gdn_send(self_numreceiveobjs);
1424 #ifdef RAWDEBUG
1425         raw_test_pass_reg(self_numreceiveobjs);
1426         raw_test_pass(0xffff);
1427 #endif
1428         return true;
1429 #elif defined THREADSIMULATE
1430         struct ___Object___ *newobj = RUNMALLOC(sizeof(struct ___Object___));
1431         // use the first four int field to hold msgtype/corenum/sendobj/receiveobj
1432         newobj->type = -1;
1433         int numofcore = pthread_getspecific(key);
1434         newobj->flag = numofcore;
1435         newobj->___cachedHash___ = thread_data_array[numofcore].numsendobjs;
1436         newobj->___cachedCode___ = thread_data_array[numofcore].numreceiveobjs;
1437
1438         // use POSIX message queue to send stall msg to startup core
1439         assert(targetcore == STARTUPCORE);
1440         mqd_t mqdnum;
1441         char corenumstr[3];
1442         int sourcelen = 0;
1443         if(targetcore < 10) {
1444                 corenumstr[0] = targetcore + '0';
1445                 corenumstr[1] = '\0';
1446                 sourcelen = 1;
1447         } else if(targetcore < 100) {
1448                 corenumstr[1] = targetcore % 10 + '0';
1449                 corenumstr[0] = (targetcore / 10) + '0';
1450                 corenumstr[2] = '\0';
1451                 sourcelen = 2;
1452         } else {
1453                 printf("Error: targetcore >= 100\n");
1454                 fflush(stdout);
1455                 exit(-1);
1456         }
1457         char * pathhead = "/msgqueue_";
1458         int targetlen = strlen(pathhead);
1459         char path[targetlen + sourcelen + 1];
1460         strcpy(path, pathhead);
1461         strncat(path, corenumstr, sourcelen);
1462         int oflags = O_WRONLY|O_NONBLOCK;
1463         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1464         mqdnum = mq_open(path, oflags, omodes, NULL);
1465         if(mqdnum==-1) {
1466                 printf("[transStallMsg, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1467                 fflush(stdout);
1468                 exit(-1);
1469         }
1470         int ret;
1471         ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1472         if(ret != 0) {
1473                 printf("[transStallMsg, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1474                 RUNFREE(newobj);
1475                 return false;
1476         } else {
1477                 printf("[transStallMsg, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1478                 printf("<transStallMsg> to %s index: %d, sendobjs: %d, receiveobjs: %d\n", path, newobj->flag, newobj->___cachedHash___, newobj->___cachedCode___);
1479                 RUNFREE(newobj);
1480                 return true;
1481         }
1482 #endif
1483 }
1484
1485 // receive object transferred from other cores
1486 // or the terminate message from other cores
1487 // NOTICE: following format is for threadsimulate version only
1488 //         RAW version please see previous description
1489 // format: type + object
1490 // type: -1--stall msg
1491 //      !-1--object
1492 // return value: 0--received an object
1493 //               1--received nothing
1494 //               2--received a Stall Msg
1495 //               3--received a lock Msg
1496 //               RAW version: -1 -- received nothing
1497 //                            otherwise -- received msg type
1498 int receiveObject() {
1499 #ifdef RAW
1500         bool deny = false;
1501         unsigned msgHdr;
1502         int self_y, self_x, target_y, target_x;
1503         int targetcore = 0;
1504         if(gdn_input_avail() == 0) {
1505 #ifdef RAWDEBUG
1506                 if(corenum < NUMCORES) {
1507                         raw_test_pass(0xd001);
1508                 }
1509 #endif
1510                 return -1;
1511         }
1512 msg:
1513 #ifdef RAWDEBUG
1514         raw_test_pass(0xcccc);
1515 #endif
1516         while((gdn_input_avail() != 0) && (msgdataindex < msglength)) {
1517                 msgdata[msgdataindex] = gdn_receive();
1518                 if(msgdataindex == 0) {
1519                         if(msgdata[0] > 2) {
1520                                 msglength = 3;
1521                         } else if(msgdata[0] > 0) {
1522                                 msglength = 4;
1523                         }
1524                 } else if((msgdataindex == 1) && (msgdata[0] == 0)) {
1525                         msglength = msgdata[msgdataindex];
1526                 }
1527 #ifdef RAWDEBUG
1528                 raw_test_pass_reg(msgdata[msgdataindex]);
1529 #endif
1530                 msgdataindex++;
1531
1532                 /*if(msgdataindex == 0) {
1533                         // type
1534                         msgtype = gdn_receive();
1535                         if(msgtype > 2) {
1536                                 msglength = 3;
1537                         } else {
1538                                 msglength = 4;
1539                         }
1540                         if(msgtype != 0) {
1541                                 msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1542                                 msgdata[msgdataindex] = msgtype;
1543                         }
1544 #ifdef RAWDEBUG
1545                         raw_test_pass_reg(msgtype);
1546 #endif
1547                 } else if((msgdataindex == 1) && (msgtype == 0)) {
1548                         // object transfer msg
1549                         msglength = gdn_receive();
1550                         msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1551                         msgdata[0] = msgtype;
1552                         msgdata[msgdataindex] = msglength;
1553 #ifdef RAWDEBUG
1554                         raw_test_pass_reg(msgdata[msgdataindex]);
1555 #endif
1556                 } else {
1557                         msgdata[msgdataindex] = gdn_receive();
1558 #ifdef RAWDEBUG
1559                         raw_test_pass_reg(msgdata[msgdataindex]);
1560 #endif
1561                 }
1562                 msgdataindex++;*/
1563         }
1564 #ifdef RAWDEBUG
1565         raw_test_pass(0xffff);
1566 #endif
1567         if(msgdataindex == msglength) {
1568                 // received a whole msg
1569                 int type, data1, data2; // will receive at least 3 words including type
1570                 type = msgdata[0];
1571                 data1 = msgdata[1];
1572                 data2 = msgdata[2];
1573                 switch(type) {
1574                         case 0: {
1575                                                 // receive a object transfer msg
1576                                                 struct transObjInfo * transObj = RUNMALLOC_I(sizeof(struct transObjInfo));
1577                                                 int k = 0;
1578                                                 if(corenum > NUMCORES - 1) {
1579                                                         raw_test_done(0xa00a);
1580                                                 }
1581                                                 // store the object and its corresponding queue info, enqueue it later
1582                                                 transObj->objptr = (void *)data2; // data1 is now size of the msg
1583                                                 transObj->length = (msglength - 3) / 2;
1584                                                 transObj->queues = RUNMALLOC_I(sizeof(int)*(msglength - 3));
1585                                                 for(k = 0; k < transObj->length; ++k) {
1586                                                         transObj->queues[2*k] = msgdata[3+2*k];
1587 #ifdef RAWDEBUG
1588                                                         raw_test_pass_reg(transObj->queues[2*k]);
1589 #endif
1590                                                         transObj->queues[2*k+1] = msgdata[3+2*k+1];
1591 #ifdef RAWDEBUG
1592                                                         raw_test_pass_reg(transObj->queues[2*k+1]);
1593 #endif
1594                                                 }
1595                                                 //memcpy(transObj->queues, msgdata[3], sizeof(int)*(msglength - 3));
1596                                                 addNewItem_I(&objqueue, (void *)transObj);
1597                                                 ++(self_numreceiveobjs);
1598 #ifdef RAWDEBUG
1599                                                 raw_test_pass(0xe881);
1600 #endif
1601                                                 /*
1602                                                 addNewItem_I(&objqueue, (void *)data2);
1603                                                 ++(self_numreceiveobjs);
1604 #ifdef RAWDEBUG
1605                                                 raw_test_pass(0xe881);
1606 #endif
1607                         */
1608                                                 break;
1609                                         }
1610                         case 1: {
1611                                                 // receive a stall msg
1612                                                 if(corenum != STARTUPCORE) {
1613                                                         // non startup core can not receive stall msg
1614                                                         // return -1
1615                                                         raw_test_done(0xa001);
1616                                                 }
1617                                                 if(data1 < NUMCORES) {
1618 #ifdef RAWDEBUG
1619                                                         raw_test_pass(0xe882);
1620 #endif
1621                                                         corestatus[data1] = 0;
1622                                                         numsendobjs[data1] = data2;
1623                                                         numreceiveobjs[data1] = msgdata[3];
1624                                                 }
1625                                                 break;
1626                                         }
1627                         case 2: {
1628                                                 // receive lock request msg
1629                                                 // for 32 bit machine, the size is always 3 words
1630                                                 //int msgsize = sizeof(int) * 3;
1631                                                 int msgsize = 3;
1632                                                 // lock request msg, handle it right now
1633                                                 // check to see if there is a lock exist in locktbl for the required obj
1634                                                 int data3 = msgdata[3];
1635                                                 deny = false;
1636                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1637                                                         // no locks for this object
1638                                                         // first time to operate on this shared object
1639                                                         // create a lock for it
1640                                                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1641 #ifdef RAWDEBUG
1642                                                         raw_test_pass(0xe883);
1643 #endif
1644                                                         if(data1 == 0) {
1645                                                                 RuntimeHashadd_I(locktbl, data2, 1);
1646                                                         } else {
1647                                                                 RuntimeHashadd_I(locktbl, data2, -1);
1648                                                         }
1649                                                 } else {
1650                                                         int rwlock_obj = 0;
1651 #ifdef RAWDEBUG
1652                                                         raw_test_pass(0xe884);
1653 #endif
1654                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1655 #ifdef RAWDEBUG
1656                                                         raw_test_pass_reg(rwlock_obj);
1657 #endif
1658                                                         if(0 == rwlock_obj) {
1659                                                                 if(data1 == 0) {
1660                                                                         rwlock_obj = 1;
1661                                                                 } else {
1662                                                                         rwlock_obj = -1;
1663                                                                 }
1664                                                                 RuntimeHashremovekey(locktbl, data2);
1665                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1666                                                         } else if((rwlock_obj > 0) && (data1 == 0)) {
1667                                                                 // read lock request and there are only read locks
1668                                                                 rwlock_obj++;
1669                                                                 RuntimeHashremovekey(locktbl, data2);
1670                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1671                                                         } else {
1672                                                                 deny = true;
1673                                                         }
1674 #ifdef RAWDEBUG
1675                                                         raw_test_pass_reg(rwlock_obj);
1676 #endif
1677                                                 }
1678                                                 targetcore = data3;
1679                                                 calCoords(corenum, &self_y, &self_x);
1680                                                 calCoords(targetcore, &target_y, &target_x);
1681                                                 // Build the message header
1682                                                 msgHdr = construct_dyn_hdr(0, msgsize, 0,               // msgsize word sent.
1683                                                                                    self_y, self_x,
1684                                                                                                    target_y, target_x);
1685                                                 gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1686 #ifdef RAWDEBUG
1687                                                 raw_test_pass(0xbbbb);
1688                                                 raw_test_pass(0xb000 + targetcore); // targetcore
1689 #endif
1690                                                 if(deny == true) {
1691                                                         // deny the lock request
1692                                                         gdn_send(4); // lock request
1693 #ifdef RAWDEBUG
1694                                                         raw_test_pass(4);
1695 #endif
1696                                                 } else {
1697                                                         // grount the lock request
1698                                                         gdn_send(3); // lock request
1699 #ifdef RAWDEBUG
1700                                                         raw_test_pass(3);
1701 #endif
1702                                                 }
1703                                                 gdn_send(data1); // lock type
1704 #ifdef RAWDEBUG
1705                                                 raw_test_pass_reg(data1);
1706 #endif
1707                                                 gdn_send(data2); // lock target
1708 #ifdef RAWDEBUG
1709                                                 raw_test_pass_reg(data2);
1710                                                 raw_test_pass(0xffff);
1711 #endif
1712                                                 break;
1713                                         }
1714                         case 3: {
1715                                                 // receive lock grount msg
1716                                                 if(corenum > NUMCORES - 1) {
1717                                                         raw_test_done(0xa00b);
1718                                                 }
1719                                                 if(lockobj == data2) {
1720                                                         lockresult = 1;
1721                                                         lockflag = true;
1722 #ifndef INTERRUPT
1723                                                         reside = false;
1724 #endif
1725                                                 } else {
1726                                                         // conflicts on lockresults
1727                                                 raw_test_done(0xa002);
1728                                                 }
1729                                                 break;
1730                                         }
1731                         case 4: {
1732                                                 // receive lock grount/deny msg
1733                                                 if(corenum > NUMCORES - 1) {
1734                                                         raw_test_done(0xa00c);
1735                                                 }
1736                                                 if(lockobj == data2) {
1737                                                         lockresult = 0;
1738                                                         lockflag = true;
1739 #ifndef INTERRUPT
1740                                                         reside = false;
1741 #endif
1742                                                 } else {
1743                                                         // conflicts on lockresults
1744                                                         raw_test_done(0xa003);
1745                                                 }
1746                                                 break;
1747                                         }
1748                         case 5: {
1749                                                 // receive lock release msg
1750                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1751                                                         // no locks for this object, something is wrong
1752                                                         raw_test_done(0xa004);
1753                                                 } else {
1754                                                         int rwlock_obj = 0;
1755                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1756 #ifdef RAWDEBUG
1757                                                         raw_test_pass(0xe885);
1758                                                         raw_test_pass_reg(rwlock_obj);
1759 #endif
1760                                                         if(data1 == 0) {
1761                                                                 rwlock_obj--;
1762                                                         } else {
1763                                                                 rwlock_obj++;
1764                                                         }
1765                                                         RuntimeHashremovekey(locktbl, data2);
1766                                                         RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1767 #ifdef RAWDEBUG
1768                                                         raw_test_pass_reg(rwlock_obj);
1769 #endif
1770                                                 }
1771                                                 break;
1772                                         }
1773                         default:
1774                                         break;
1775                 }
1776                 //RUNFREE(msgdata);
1777                 //msgdata = NULL;
1778                 for(msgdataindex--;msgdataindex > 0; --msgdataindex) {
1779                         msgdata[msgdataindex] = -1;
1780                 }
1781                 msgtype = -1;
1782                 //msgdataindex = 0;
1783                 msglength = 30;
1784 #ifdef RAWDEBUG
1785                 raw_test_pass(0xe886);
1786 #endif
1787                 if(gdn_input_avail() != 0) {
1788                         goto msg;
1789                 }
1790                 return type;
1791         } else {
1792                 // not a whole msg
1793 #ifdef RAWDEBUG
1794                 raw_test_pass(0xe887);
1795 #endif
1796                 return -2;
1797         }
1798 #elif defined THREADSIMULATE
1799         int numofcore = pthread_getspecific(key);
1800         // use POSIX message queue to transfer object
1801         int msglen = 0;
1802         struct mq_attr mqattr;
1803         mq_getattr(mqd[numofcore], &mqattr);
1804         void * msgptr =RUNMALLOC(mqattr.mq_msgsize);
1805         msglen=mq_receive(mqd[numofcore], msgptr, mqattr.mq_msgsize, NULL); // receive the object into the queue
1806         if(-1 == msglen) {
1807                 // no msg
1808                 free(msgptr);
1809                 return 1;
1810         }
1811         //printf("msg: %s\n",msgptr);
1812         if(((int*)msgptr)[0] == -1) {
1813                 // StallMsg
1814                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1815                 int index = tmpptr->flag;
1816                 corestatus[index] = 0;
1817                 numsendobjs[index] = tmpptr->___cachedHash___;
1818                 numreceiveobjs[index] = tmpptr->___cachedCode___;
1819                 printf("<receiveObject> index: %d, sendobjs: %d, reveiveobjs: %d\n", index, numsendobjs[index], numreceiveobjs[index]);
1820                 free(msgptr);
1821                 return 2;
1822         } /*else if(((int*)msgptr)[0] == -2) {
1823                 // terminate msg
1824                 return 3;
1825         } */else {
1826                 // an object
1827                 if(numofcore == STARTUPCORE) {
1828                         ++(numreceiveobjs[numofcore]);
1829                 } else {
1830                         ++(thread_data_array[numofcore].numreceiveobjs);
1831                 }
1832                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1833                 struct transObjInfo * transObj = (struct transObjInfo *)tmpptr->original;
1834                 tmpptr = (struct ___Object___ *)(transObj->objptr);
1835                 int type = tmpptr->type;
1836                 int size=classsize[type];
1837                 struct ___Object___ * newobj=RUNMALLOC(size);
1838                 memcpy(newobj, tmpptr, size);
1839                 if(0 == newobj->isolate) {
1840                         newobj->original=tmpptr;
1841                 }
1842                 RUNFREE(msgptr);
1843                 tmpptr = NULL;
1844                 int k = 0;
1845                 for(k = 0; k < transObj->length; ++k) {
1846                         int taskindex = transObj->queues[2 * k];
1847                         int paramindex = transObj->queues[2 * k + 1];
1848                         struct parameterwrapper ** queues = &(paramqueues[numofcore][taskindex][paramindex]);
1849                         enqueueObject(newobj, queues, 1);
1850                 }
1851                 RUNFREE(transObj->queues);
1852                 RUNFREE(transObj);
1853                 return 0;
1854         }
1855 #endif
1856 }
1857
1858 bool getreadlock(void * ptr) {
1859 #ifdef RAW
1860         unsigned msgHdr;
1861         int self_y, self_x, target_y, target_x;
1862         int targetcore = ((int)ptr >> 5) % TOTALCORE;
1863         // for 32 bit machine, the size is always 4 words
1864         //int msgsize = sizeof(int) * 4;
1865         int msgsize = 4;
1866
1867         lockobj = (int)ptr;
1868         lockflag = false;
1869 #ifndef INTERRUPT
1870         reside = false;
1871 #endif
1872         lockresult = 0;
1873
1874         if(targetcore == corenum) {
1875                 // reside on this core
1876                 bool deny = false;
1877 #ifdef INTERRUPT
1878                 raw_user_interrupts_off();
1879 #endif
1880                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1881                         // no locks for this object
1882                         // first time to operate on this shared object
1883                         // create a lock for it
1884                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1885                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
1886                 } else {
1887                         int rwlock_obj = 0;
1888                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
1889                         if(-1 != rwlock_obj) {
1890                                 rwlock_obj++;
1891                                 RuntimeHashremovekey(locktbl, (int)ptr);
1892                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
1893                         } else {
1894                                 deny = true;
1895                         }
1896                 }
1897 #ifdef INTERRUPT
1898                 raw_user_interrupts_on();
1899 #endif
1900                 if(lockobj == (int)ptr) {
1901                         if(deny) {
1902                                 lockresult = 0;
1903                         } else {
1904                                 lockresult = 1;
1905                         }
1906                         lockflag = true;
1907 #ifndef INTERRUPT
1908                         reside = true;
1909 #endif
1910                 } else {
1911                         // conflicts on lockresults
1912                         raw_test_done(0xa005);
1913                 }
1914                 return true;
1915         }
1916
1917         calCoords(corenum, &self_y, &self_x);
1918         calCoords(targetcore, &target_y, &target_x);
1919         // Build the message header
1920     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1921                                            self_y, self_x,
1922                                                            target_y, target_x);
1923         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1924 #ifdef RAWDEBUG
1925         raw_test_pass(0xbbbb);
1926         raw_test_pass(0xb000 + targetcore); // targetcore
1927 #endif
1928     gdn_send(2); // lock request
1929  #ifdef RAWDEBUG
1930     raw_test_pass(2);
1931 #endif
1932         gdn_send(0); // read lock
1933 #ifdef RAWDEBUG
1934     raw_test_pass(0);
1935 #endif
1936     gdn_send(ptr);
1937 #ifdef RAWDEBUG
1938         raw_test_pass_reg(ptr);
1939 #endif
1940         gdn_send(corenum);
1941 #ifdef RAWDEBUG
1942         raw_test_pass_reg(corenum);
1943         raw_test_pass(0xffff);
1944 #endif
1945         return true;
1946 #elif defined THREADSIMULATE
1947         int numofcore = pthread_getspecific(key);
1948
1949         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
1950         printf("[getreadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1951         if(0 != rc) {
1952                 return false;
1953         }
1954         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1955                 // no locks for this object
1956                 // first time to operate on this shared object
1957                 // create a lock for it
1958                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1959                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1960                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
1961                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
1962                 rc = pthread_rwlock_init(rwlock, NULL);
1963                 printf("[getreadlock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1964                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
1965                 printf("[getreadlock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1966                 if(0 != rc) {
1967                         RUNFREE(rwlock);
1968                         return false;
1969                 } else {
1970                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1971                                 // check again
1972                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
1973                         } else {
1974                                 RUNFREE(rwlock);
1975                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
1976                         }
1977                         rc = pthread_rwlock_unlock(&rwlock_tbl);
1978                                 printf("[getreadlock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1979                 }
1980                 rc = pthread_rwlock_tryrdlock(rwlock);
1981                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1982                 if(0 != rc) {
1983                         return false;
1984                 } else {
1985                         return true;
1986                 }
1987         } else {
1988                 pthread_rwlock_t* rwlock_obj = NULL;
1989                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
1990                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1991                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1992                 int rc_obj = pthread_rwlock_tryrdlock(rwlock_obj);
1993                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
1994                 if(0 != rc_obj) {
1995                         return false;
1996                 } else {
1997                         return true;
1998                 }
1999         }
2000 #endif
2001 }
2002
2003 void releasereadlock(void * ptr) {
2004 #ifdef RAW
2005         unsigned msgHdr;
2006         int self_y, self_x, target_y, target_x;
2007         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2008         // for 32 bit machine, the size is always 3 words
2009         //int msgsize = sizeof(int) * 3;
2010         int msgsize = 3;
2011
2012         if(targetcore == corenum) {
2013 #ifdef INTERRUPT
2014                 raw_user_interrupts_off();
2015 #endif
2016                 // reside on this core
2017                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2018                         // no locks for this object, something is wrong
2019                         raw_test_done(0xa006);
2020                 } else {
2021                         int rwlock_obj = 0;
2022                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2023                         rwlock_obj--;
2024                         RuntimeHashremovekey(locktbl, (int)ptr);
2025                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2026                 }
2027 #ifdef INTERRUPT
2028                 raw_user_interrupts_on();
2029 #endif
2030                 return;
2031         }
2032
2033         calCoords(corenum, &self_y, &self_x);
2034         calCoords(targetcore, &target_y, &target_x);
2035         // Build the message header
2036     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2037                                            self_y, self_x,
2038                                                            target_y, target_x);
2039         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2040 #ifdef RAWDEBUG
2041         raw_test_pass(0xbbbb);
2042         raw_test_pass(0xb000 + targetcore); // targetcore
2043 #endif
2044     gdn_send(5); // lock release
2045 #ifdef RAWDEBUG
2046     raw_test_pass(5);
2047 #endif
2048         gdn_send(0); // read lock
2049 #ifdef RAWDEBUG
2050     raw_test_pass(0);
2051 #endif
2052     gdn_send(ptr);
2053 #ifdef RAWDEBUG
2054         raw_test_pass_reg(ptr);
2055         raw_test_pass(0xffff);
2056 #endif
2057 #elif defined THREADSIMULATE
2058         int numofcore = pthread_getspecific(key);
2059         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
2060         printf("[releasereadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2061         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2062                 printf("[releasereadlock, %d] Error: try to release a lock without previously grab it\n", numofcore);
2063                 exit(-1);
2064         }
2065         pthread_rwlock_t* rwlock_obj = NULL;
2066         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2067         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
2068         printf("[releasereadlock, %d] unlocked object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2069         rc = pthread_rwlock_unlock(&rwlock_tbl);
2070         printf("[releasereadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2071 #endif
2072 }
2073
2074 #ifdef RAW
2075 bool getreadlock_I(void * ptr) {
2076         unsigned msgHdr;
2077         int self_y, self_x, target_y, target_x;
2078         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2079         // for 32 bit machine, the size is always 4 words
2080         //int msgsize = sizeof(int) * 4;
2081         int msgsize = 4;
2082
2083         lockobj = (int)ptr;
2084         lockflag = false;
2085 #ifndef INTERRUPT
2086         reside = false;
2087 #endif
2088         lockresult = 0;
2089
2090         if(targetcore == corenum) {
2091                 // reside on this core
2092                 bool deny = false;
2093                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2094                         // no locks for this object
2095                         // first time to operate on this shared object
2096                         // create a lock for it
2097                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
2098                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
2099                 } else {
2100                         int rwlock_obj = 0;
2101                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2102                         if(-1 != rwlock_obj) {
2103                                 rwlock_obj++;
2104                                 RuntimeHashremovekey(locktbl, (int)ptr);
2105                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2106                         } else {
2107                                 deny = true;
2108                         }
2109                 }
2110                 if(lockobj == (int)ptr) {
2111                         if(deny) {
2112                                 lockresult = 0;
2113                         } else {
2114                                 lockresult = 1;
2115                         }
2116                         lockflag = true;
2117 #ifndef INTERRUPT
2118                         reside = true;
2119 #endif
2120                 } else {
2121                         // conflicts on lockresults
2122                         raw_test_done(0xa005);
2123                 }
2124                 return true;
2125         }
2126
2127         calCoords(corenum, &self_y, &self_x);
2128         calCoords(targetcore, &target_y, &target_x);
2129         // Build the message header
2130     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2131                                            self_y, self_x,
2132                                                            target_y, target_x);
2133         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2134 #ifdef RAWDEBUG
2135         raw_test_pass(0xbbbb);
2136         raw_test_pass(0xb000 + targetcore); // targetcore
2137 #endif
2138     gdn_send(2); // lock request
2139 #ifdef RAWDEBUG
2140     raw_test_pass(2);
2141 #endif
2142         gdn_send(0); // read lock
2143 #ifdef RAWDEBUG
2144     raw_test_pass(0);
2145 #endif
2146     gdn_send(ptr);
2147 #ifdef RAWDEBUG
2148         raw_test_pass_reg(ptr);
2149 #endif
2150         gdn_send(corenum);
2151 #ifdef RAWDEBUG
2152         raw_test_pass_reg(corenum);
2153         raw_test_pass(0xffff);
2154 #endif
2155         return true;
2156 }
2157
2158 void releasereadlock_I(void * ptr) {
2159         unsigned msgHdr;
2160         int self_y, self_x, target_y, target_x;
2161         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2162         // for 32 bit machine, the size is always 3 words
2163         //int msgsize = sizeof(int) * 3;
2164         int msgsize = 3;
2165
2166         if(targetcore == corenum) {
2167                 // reside on this core
2168                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2169                         // no locks for this object, something is wrong
2170                         raw_test_done(0xa006);
2171                 } else {
2172                         int rwlock_obj = 0;
2173                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2174                         rwlock_obj--;
2175                         RuntimeHashremovekey(locktbl, (int)ptr);
2176                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2177                 }
2178                 return;
2179         }
2180
2181         calCoords(corenum, &self_y, &self_x);
2182         calCoords(targetcore, &target_y, &target_x);
2183         // Build the message header
2184     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2185                                            self_y, self_x,
2186                                                            target_y, target_x);
2187         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2188 #ifdef RAWDEBUG
2189         raw_test_pass(0xbbbb);
2190         raw_test_pass(0xb000 + targetcore); // targetcore
2191 #endif
2192     gdn_send(5); // lock release
2193 #ifdef RAWDEBUG
2194     raw_test_pass(5);
2195 #endif
2196         gdn_send(0); // read lock
2197 #ifdef RAWDEBUG
2198     raw_test_pass(0);
2199 #endif
2200     gdn_send(ptr);
2201 #ifdef RAWDEBUG
2202         raw_test_pass_reg(ptr);
2203         raw_test_pass(0xffff);
2204 #endif
2205 }
2206 #endif
2207
2208 // not reentrant
2209 bool getwritelock(void * ptr) {
2210 #ifdef RAW
2211         unsigned msgHdr;
2212         int self_y, self_x, target_y, target_x;
2213         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2214         // for 32 bit machine, the size is always 4 words
2215         //int msgsize = sizeof(int) * 4;
2216         int msgsize= 4;
2217         int tc = TOTALCORE;
2218 #ifdef INTERRUPT
2219         //raw_user_interrupts_off();
2220 #endif
2221         //targetcore = ((int)ptr) % tc;
2222 #ifdef INTERRUPT
2223         //raw_user_interrupts_on();
2224 #endif
2225
2226 #ifdef RAWDEBUG
2227         raw_test_pass(0xe551);
2228         raw_test_pass_reg(ptr);
2229         raw_test_pass_reg(targetcore);
2230         raw_test_pass_reg(tc);
2231 #endif
2232
2233         lockobj = (int)ptr;
2234         lockflag = false;
2235 #ifndef INTERRUPT
2236         reside = false;
2237 #endif
2238         lockresult = 0;
2239
2240         if(targetcore == corenum) {
2241                 // reside on this core
2242                 bool deny = false;
2243 #ifdef INTERRUPT
2244                 raw_user_interrupts_off();
2245 #endif
2246                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2247                         // no locks for this object
2248                         // first time to operate on this shared object
2249                         // create a lock for it
2250                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
2251 #ifdef RAWDEBUG
2252                         raw_test_pass(0xe552);
2253 #endif
2254                         RuntimeHashadd_I(locktbl, (int)ptr, -1);
2255                 } else {
2256                         int rwlock_obj = 0;
2257                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2258 #ifdef RAWDEBUG
2259                         raw_test_pass(0xe553);
2260                         raw_test_pass_reg(rwlock_obj);
2261 #endif
2262                         if(0 == rwlock_obj) {
2263                                 rwlock_obj = -1;
2264                                 RuntimeHashremovekey(locktbl, (int)ptr);
2265                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2266                         } else {
2267                                 deny = true;
2268                         }
2269                 }
2270 #ifdef INTERRUPT
2271                 raw_user_interrupts_on();
2272 #endif
2273 #ifdef RAWDEBUG
2274                 raw_test_pass(0xe554);
2275                 raw_test_pass_reg(lockresult);
2276 #endif
2277                 if(lockobj == (int)ptr) {
2278                         if(deny) {
2279                                 lockresult = 0;
2280 #ifdef RAWDEBUG
2281                                 raw_test_pass(0);
2282 #endif
2283                         } else {
2284                                 lockresult = 1;
2285 #ifdef RAWDEBUG
2286                 raw_test_pass(1);
2287 #endif
2288                         }
2289                         lockflag = true;
2290 #ifndef INTERRUPT
2291                         reside = true;
2292 #endif
2293                 } else {
2294                         // conflicts on lockresults
2295                         raw_test_done(0xa007);
2296                 }
2297                 return true;
2298         }
2299
2300 #ifdef RAWDEBUG
2301         raw_test_pass(0xe555);
2302 #endif
2303         calCoords(corenum, &self_y, &self_x);
2304         calCoords(targetcore, &target_y, &target_x);
2305         // Build the message header
2306     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2307                                            self_y, self_x,
2308                                                            target_y, target_x);
2309         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2310 #ifdef RAWDEBUG
2311         raw_test_pass(0xbbbb);
2312         raw_test_pass(0xb000 + targetcore); // targetcore
2313 #endif
2314     gdn_send(2); // lock request
2315 #ifdef RAWDEBUG
2316     raw_test_pass(2);
2317 #endif
2318         gdn_send(1); // write lock
2319 #ifdef RAWDEBUG
2320     raw_test_pass(1);
2321 #endif
2322     gdn_send(ptr);
2323 #ifdef RAWDEBUG
2324         raw_test_pass_reg(ptr);
2325 #endif
2326         gdn_send(corenum);
2327 #ifdef RAWDEBUG
2328         raw_test_pass_reg(corenum);
2329         raw_test_pass(0xffff);
2330 #endif
2331         return true;
2332 #elif defined THREADSIMULATE
2333         int numofcore = pthread_getspecific(key);
2334
2335         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
2336         printf("[getwritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2337         if(0 != rc) {
2338                 return false;
2339         }
2340         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2341                 // no locks for this object
2342                 // first time to operate on this shared object
2343                 // create a lock for it
2344                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2345                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2346                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
2347                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
2348                 rc = pthread_rwlock_init(rwlock, NULL);
2349                 printf("[getwritelock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2350                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
2351                 printf("[getwritelock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2352                 if(0 != rc) {
2353                         pthread_rwlock_destroy(rwlock);
2354                         RUNFREE(rwlock);
2355                         return false;
2356                 } else {
2357                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2358                                 // check again
2359                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
2360                         } else {
2361                                 pthread_rwlock_destroy(rwlock);
2362                                 RUNFREE(rwlock);
2363                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
2364                         }
2365                         rc = pthread_rwlock_unlock(&rwlock_tbl);
2366                         printf("[getwritelock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2367                 }
2368                 rc = pthread_rwlock_trywrlock(rwlock);
2369                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2370                 if(0 != rc) {
2371                         return false;
2372                 } else {
2373                         return true;
2374                 }
2375         } else {
2376                 pthread_rwlock_t* rwlock_obj = NULL;
2377                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2378                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2379                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2380                 int rc_obj = pthread_rwlock_trywrlock(rwlock_obj);
2381                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2382                 if(0 != rc_obj) {
2383                         return false;
2384                 } else {
2385                         return true;
2386                 }
2387         }
2388
2389 #endif
2390 }
2391
2392 void releasewritelock(void * ptr) {
2393 #ifdef RAW
2394         unsigned msgHdr;
2395         int self_y, self_x, target_y, target_x;
2396         int targetcore = ((int)ptr >> 5) % TOTALCORE;
2397         // for 32 bit machine, the size is always 3 words
2398         //int msgsize = sizeof(int) * 3;
2399         int msgsize = 3;
2400
2401         if(targetcore == corenum) {
2402 #ifdef INTERRUPT
2403                 raw_user_interrupts_off();
2404 #endif
2405                 // reside on this core
2406                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2407                         // no locks for this object, something is wrong
2408                         raw_test_done(0xa008);
2409                 } else {
2410                         int rwlock_obj = 0;
2411 #ifdef RAWDEBUG
2412                         raw_test_pass(0xe662);
2413 #endif
2414                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2415 #ifdef RAWDEBUG
2416                         raw_test_pass_reg(rwlock_obj);
2417 #endif
2418                         rwlock_obj++;
2419                         RuntimeHashremovekey(locktbl, (int)ptr);
2420                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2421 #ifdef RAWDEBUG
2422                         raw_test_pass_reg(rwlock_obj);
2423 #endif
2424                 }
2425 #ifdef INTERRUPT
2426                 raw_user_interrupts_on();
2427 #endif
2428                 return;
2429         }
2430
2431 #ifdef RAWDEBUG
2432         raw_test_pass(0xe663);
2433 #endif
2434         calCoords(corenum, &self_y, &self_x);
2435         calCoords(targetcore, &target_y, &target_x);
2436         // Build the message header
2437     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2438                                            self_y, self_x,
2439                                                            target_y, target_x);
2440         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2441 #ifdef RAWDEBUG
2442         raw_test_pass(0xbbbb);
2443         raw_test_pass(0xb000 + targetcore);
2444 #endif
2445     gdn_send(5); // lock release
2446  #ifdef RAWDEBUG
2447     raw_test_pass(5);
2448 #endif
2449         gdn_send(1); // write lock
2450 #ifdef RAWDEBUG
2451     raw_test_pass(1);
2452 #endif
2453     gdn_send(ptr);
2454 #ifdef RAWDEBUG
2455         raw_test_pass_reg(ptr);
2456         raw_test_pass(0xffff);
2457 #endif
2458 #elif defined THREADSIMULATE
2459         int numofcore = pthread_getspecific(key);
2460         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
2461         printf("[releasewritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2462         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2463                 printf("[releasewritelock, %d] Error: try to release a lock without previously grab it\n", numofcore);
2464                 exit(-1);
2465         }
2466         pthread_rwlock_t* rwlock_obj = NULL;
2467         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2468         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
2469         printf("[releasewritelock, %d] unlocked object %d: %d error:\n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2470         rc = pthread_rwlock_unlock(&rwlock_tbl);
2471         printf("[releasewritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2472 #endif
2473 }
2474
2475 int enqueuetasks(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2476   void * taskpointerarray[MAXTASKPARAMS];
2477   int j;
2478   int numparams=parameter->task->numParameters;
2479   int numiterators=parameter->task->numTotal-1;
2480   int retval=1;
2481   int addnormal=1;
2482   int adderror=1;
2483
2484   struct taskdescriptor * task=parameter->task;
2485
2486         ObjectHashadd(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2487
2488   /* Add enqueued object to parameter vector */
2489   taskpointerarray[parameter->slot]=ptr;
2490
2491   /* Reset iterators */
2492   for(j=0;j<numiterators;j++) {
2493     toiReset(&parameter->iterators[j]);
2494   }
2495
2496   /* Find initial state */
2497   for(j=0;j<numiterators;j++) {
2498   backtrackinit:
2499     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2500       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2501     else if (j>0) {
2502       /* Need to backtrack */
2503       toiReset(&parameter->iterators[j]);
2504       j--;
2505       goto backtrackinit;
2506     } else {
2507       /* Nothing to enqueue */
2508       return retval;
2509     }
2510   }
2511
2512
2513   while(1) {
2514     /* Enqueue current state */
2515     int launch = 0;
2516     struct taskparamdescriptor *tpd=RUNMALLOC(sizeof(struct taskparamdescriptor));
2517     tpd->task=task;
2518     tpd->numParameters=numiterators+1;
2519     tpd->parameterArray=RUNMALLOC(sizeof(void *)*(numiterators+1));
2520     for(j=0;j<=numiterators;j++){
2521       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2522     }
2523     /* Enqueue task */
2524     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2525       genputtable(activetasks, tpd, tpd);
2526     } else {
2527       RUNFREE(tpd->parameterArray);
2528       RUNFREE(tpd);
2529     }
2530
2531     /* This loop iterates to the next parameter combination */
2532     if (numiterators==0)
2533       return retval;
2534
2535     for(j=numiterators-1; j<numiterators;j++) {
2536     backtrackinc:
2537       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2538         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2539       else if (j>0) {
2540         /* Need to backtrack */
2541         toiReset(&parameter->iterators[j]);
2542         j--;
2543         goto backtrackinc;
2544       } else {
2545         /* Nothing more to enqueue */
2546         return retval;
2547       }
2548     }
2549   }
2550   return retval;
2551 }
2552
2553 #ifdef RAW
2554 int enqueuetasks_I(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2555   void * taskpointerarray[MAXTASKPARAMS];
2556   int j;
2557   int numparams=parameter->task->numParameters;
2558   int numiterators=parameter->task->numTotal-1;
2559   int retval=1;
2560   int addnormal=1;
2561   int adderror=1;
2562
2563   struct taskdescriptor * task=parameter->task;
2564
2565         ObjectHashadd_I(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2566
2567   /* Add enqueued object to parameter vector */
2568   taskpointerarray[parameter->slot]=ptr;
2569
2570   /* Reset iterators */
2571   for(j=0;j<numiterators;j++) {
2572     toiReset(&parameter->iterators[j]);
2573   }
2574
2575   /* Find initial state */
2576   for(j=0;j<numiterators;j++) {
2577   backtrackinit:
2578     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2579       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2580     else if (j>0) {
2581       /* Need to backtrack */
2582       toiReset(&parameter->iterators[j]);
2583       j--;
2584       goto backtrackinit;
2585     } else {
2586       /* Nothing to enqueue */
2587       return retval;
2588     }
2589   }
2590
2591   while(1) {
2592     /* Enqueue current state */
2593     int launch = 0;
2594     struct taskparamdescriptor *tpd=RUNMALLOC_I(sizeof(struct taskparamdescriptor));
2595     tpd->task=task;
2596     tpd->numParameters=numiterators+1;
2597     tpd->parameterArray=RUNMALLOC_I(sizeof(void *)*(numiterators+1));
2598     for(j=0;j<=numiterators;j++){
2599       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2600     }
2601     /* Enqueue task */
2602     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2603       genputtable_I(activetasks, tpd, tpd);
2604     } else {
2605       RUNFREE(tpd->parameterArray);
2606       RUNFREE(tpd);
2607     }
2608
2609     /* This loop iterates to the next parameter combination */
2610     if (numiterators==0)
2611       return retval;
2612
2613     for(j=numiterators-1; j<numiterators;j++) {
2614     backtrackinc:
2615       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2616         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2617       else if (j>0) {
2618         /* Need to backtrack */
2619         toiReset(&parameter->iterators[j]);
2620         j--;
2621         goto backtrackinc;
2622       } else {
2623         /* Nothing more to enqueue */
2624         return retval;
2625       }
2626     }
2627   }
2628   return retval;
2629 }
2630 #endif
2631
2632 /* Handler for signals. The signals catch null pointer errors and
2633    arithmatic errors. */
2634 #ifndef RAW
2635 void myhandler(int sig, siginfo_t *info, void *uap) {
2636   sigset_t toclear;
2637 #ifdef DEBUG
2638   printf("sig=%d\n",sig);
2639   printf("signal\n");
2640 #endif
2641   sigemptyset(&toclear);
2642   sigaddset(&toclear, sig);
2643   sigprocmask(SIG_UNBLOCK, &toclear,NULL);
2644   longjmp(error_handler,1);
2645 }
2646 #endif
2647
2648 fd_set readfds;
2649 int maxreadfd;
2650 struct RuntimeHash *fdtoobject;
2651
2652 void addreadfd(int fd) {
2653   if (fd>=maxreadfd)
2654     maxreadfd=fd+1;
2655   FD_SET(fd, &readfds);
2656 }
2657
2658 void removereadfd(int fd) {
2659   FD_CLR(fd, &readfds);
2660   if (maxreadfd==(fd+1)) {
2661     maxreadfd--;
2662     while(maxreadfd>0&&!FD_ISSET(maxreadfd-1, &readfds))
2663       maxreadfd--;
2664   }
2665 }
2666
2667 #ifdef PRECISE_GC
2668 #define OFFSET 2
2669 #else
2670 #define OFFSET 0
2671 #endif
2672
2673 void executetasks() {
2674   void * taskpointerarray[MAXTASKPARAMS+OFFSET];
2675   int numparams=0;
2676   int numtotal=0;
2677   struct ___Object___ * tmpparam = NULL;
2678   struct parameterdescriptor * pd=NULL;
2679   struct parameterwrapper *pw=NULL;
2680   int j = 0;
2681   int x = 0;
2682   bool lock = true;
2683
2684 #ifdef RAW
2685   int grount = 0;
2686   int andmask=0;
2687   int checkmask=0;
2688 #ifdef RAWDEBUG
2689   raw_test_pass(0xe991);
2690 #endif
2691 #endif
2692
2693 #ifndef RAW
2694   /* Set up signal handlers */
2695   struct sigaction sig;
2696   sig.sa_sigaction=&myhandler;
2697   sig.sa_flags=SA_SIGINFO;
2698   sigemptyset(&sig.sa_mask);
2699
2700   /* Catch bus errors, segmentation faults, and floating point exceptions*/
2701   sigaction(SIGBUS,&sig,0);
2702   sigaction(SIGSEGV,&sig,0);
2703   sigaction(SIGFPE,&sig,0);
2704   sigaction(SIGPIPE,&sig,0);
2705 #endif
2706
2707 #ifndef RAW
2708   /* Zero fd set */
2709   FD_ZERO(&readfds);
2710 #endif
2711   maxreadfd=0;
2712 #ifndef RAW
2713   fdtoobject=allocateRuntimeHash(100);
2714 #endif
2715
2716 #ifndef RAW
2717   /* Map first block of memory to protected, anonymous page */
2718   mmap(0, 0x1000, 0, MAP_SHARED|MAP_FIXED|MAP_ANON, -1, 0);
2719 #endif
2720
2721   newtask:
2722   while((hashsize(activetasks)>0)||(maxreadfd>0)) {
2723
2724 #ifdef RAW
2725 #ifdef RAWDEBUG
2726           raw_test_pass(0xe992);
2727 #endif
2728 #else
2729     /* Check if any filedescriptors have IO pending */
2730     if (maxreadfd>0) {
2731       int i;
2732       struct timeval timeout={0,0};
2733       fd_set tmpreadfds;
2734       int numselect;
2735       tmpreadfds=readfds;
2736       numselect=select(maxreadfd, &tmpreadfds, NULL, NULL, &timeout);
2737       if (numselect>0) {
2738         /* Process ready fd's */
2739         int fd;
2740         for(fd=0;fd<maxreadfd;fd++) {
2741           if (FD_ISSET(fd, &tmpreadfds)) {
2742             /* Set ready flag on object */
2743             void * objptr;
2744             //      printf("Setting fd %d\n",fd);
2745             if (RuntimeHashget(fdtoobject, fd,(int *) &objptr)) {
2746               if(intflagorand(objptr,1,0xFFFFFFFF)) { /* Set the first flag to 1 */
2747                          enqueueObject(objptr, NULL, 0);
2748                   }
2749             }
2750           }
2751         }
2752       }
2753     }
2754 #endif
2755
2756     /* See if there are any active tasks */
2757     if (hashsize(activetasks)>0) {
2758       int i;
2759       currtpd=(struct taskparamdescriptor *) getfirstkey(activetasks);
2760       genfreekey(activetasks, currtpd);
2761
2762       /* Check if this task has failed, allow a task that contains optional objects to fire */
2763       /*if (gencontains(failedtasks, currtpd)) {
2764         // Free up task parameter descriptor
2765         RUNFREE(currtpd->parameterArray);
2766         RUNFREE(currtpd);
2767         goto newtask;
2768       }*/
2769       numparams=currtpd->task->numParameters;
2770       numtotal=currtpd->task->numTotal;
2771
2772 #ifdef THREADSIMULATE
2773           int isolateflags[numparams];
2774 #endif
2775       /* Make sure that the parameters are still in the queues */
2776       for(i=0;i<numparams;i++) {
2777         void * parameter=currtpd->parameterArray[i];
2778 #ifdef RAW
2779 #ifdef RAWDEBUG
2780         raw_test_pass(0xe993);
2781 #endif
2782
2783         if(((struct ___Object___ *)parameter)->type == STARTUPTYPE) {
2784                         lock = false;
2785         taskpointerarray[i+OFFSET]=parameter;
2786         goto execute;
2787         }
2788         // require locks for this parameter if it is not a startup object
2789         getwritelock(parameter);
2790         grount = 0;
2791
2792 #ifdef INTERRUPT
2793         raw_user_interrupts_off();
2794 #endif
2795         while(!lockflag) {
2796                 receiveObject();
2797         }
2798 #ifndef INTERRUPT
2799         if(reside) {
2800                 while(receiveObject() != -1) {
2801                 }
2802         }
2803 #endif
2804         grount = lockresult;
2805
2806         lockresult = 0;
2807         lockobj = 0;
2808         lockflag = false;
2809 #ifndef INTERRUPT
2810         reside = false;
2811 #endif
2812 #ifdef INTERRUPT
2813         raw_user_interrupts_on();
2814 #endif
2815
2816         if(grount == 0) {
2817 #ifdef RAWDEBUG
2818                 raw_test_pass(0xe994);
2819 #endif
2820                 // can not get the lock, try later
2821                 for(j = 0; j < i; ++j) {
2822                         releasewritelock(taskpointerarray[j+OFFSET]);
2823                 }
2824                 genputtable(activetasks, currtpd, currtpd);
2825                 if(hashsize(activetasks) == 1) {
2826                         // only one task right now, wait a little while before next try
2827                         int halt = 10000;
2828                         while(halt--){}
2829                 }
2830                 goto newtask;
2831         }
2832         // flush the object
2833         {
2834                 raw_invalidate_cache_range((int)parameter, classsize[((struct ___Object___ *)parameter)->type]);
2835                 /*int tmp = 0;
2836                 for(tmp = 0; tmp < classsize[((struct ___Object___ *)parameter)->type]; ++tmp) {
2837                         invalidateAddr(parameter + tmp);
2838                 }*/
2839         }
2840 #endif
2841         tmpparam = (struct ___Object___ *)parameter;
2842 #ifdef THREADSIMULATE
2843         if(((struct ___Object___ *)parameter)->type == STARTUPTYPE) {
2844                         lock = false;
2845         taskpointerarray[i+OFFSET]=parameter;
2846         goto execute;
2847         }
2848         if(0 == tmpparam->isolate) {
2849                 isolateflags[i] = 0;
2850                 // shared object, need to flush with current value
2851                 //if(!getreadlock(tmpparam->original)) {
2852                 //      // fail to get read lock of the original object, try this task later
2853                 if(!getwritelock(tmpparam->original)) {
2854                         // fail to get write lock, release all obtained locks and try this task later
2855                         int j = 0;
2856                         for(j = 0; j < i; ++j) {
2857                                 if(0 == isolateflags[j]) {
2858                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2859                                 }
2860                         }
2861                         genputtable(activetasks, currtpd, currtpd);
2862                         goto newtask;
2863                 }
2864                 if(tmpparam->version != tmpparam->original->version) {
2865                         // some task on another core has changed this object
2866                         // flush this object
2867                         //memcpy(tmpparam, tmpparam->original, classsize[tmpparam->type]);
2868                         // release all obtained locks
2869                         int j = 0;
2870                         for(j = 0; j < i; ++j) {
2871                                 if(0 == isolateflags[j]) {
2872                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2873                                 }
2874                         }
2875                         releasewritelock(tmpparam->original);
2876
2877                         // dequeue this object
2878                         int numofcore = pthread_getspecific(key);
2879                         struct parameterwrapper ** queues = objectqueues[numofcore][tmpparam->type];
2880                         int length = numqueues[numofcore][tmpparam->type];
2881                         for(j = 0; j < length; ++j) {
2882                                 struct parameterwrapper * pw = queues[j];
2883                                 if(ObjectHashcontainskey(pw->objectset, (int)tmpparam)) {
2884                                         int next;
2885                                         int UNUSED, UNUSED2;
2886                                         int * enterflags;
2887                                         ObjectHashget(pw->objectset, (int) tmpparam, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2888                                         ObjectHashremove(pw->objectset, (int)tmpparam);
2889                                         if (enterflags!=NULL)
2890                                                 free(enterflags);
2891                                 }
2892                         }
2893                         // try to enqueue it again to check if it feeds other tasks;
2894                         //enqueueObject(tmpparam, NULL, 0);
2895                         // Free up task parameter descriptor
2896                         RUNFREE(currtpd->parameterArray);
2897                         RUNFREE(currtpd);
2898                         goto newtask;
2899                 }
2900         } else {
2901                 isolateflags[i] = 1;
2902         }
2903 #endif
2904         pd=currtpd->task->descriptorarray[i];
2905         pw=(struct parameterwrapper *) pd->queue;
2906         /* Check that object is still in queue */
2907         {
2908           if (!ObjectHashcontainskey(pw->objectset, (int) parameter)) {
2909 #ifdef RAWDEBUG
2910                   raw_test_pass(0xe995);
2911 #endif
2912                 // release grabbed locks
2913                 for(j = 0; j < i; ++j) {
2914                         releasewritelock(taskpointerarray[j+OFFSET]);
2915                 }
2916                 releasewritelock(parameter);
2917             RUNFREE(currtpd->parameterArray);
2918             RUNFREE(currtpd);
2919             goto newtask;
2920           }
2921         }
2922 #ifdef RAW
2923         /* Check if the object's flags still meets requirements */
2924         {
2925                 int tmpi = 0;
2926                 bool ismet = false;
2927                 for(tmpi = 0; tmpi < pw->numberofterms; ++tmpi) {
2928                         andmask=pw->intarray[tmpi*2];
2929                         checkmask=pw->intarray[tmpi*2+1];
2930 #ifdef RAWDEBUG
2931                         raw_test_pass(0xdd000000 + andmask);
2932                         raw_test_pass_reg((int)parameter);
2933                         raw_test_pass(0xdd000000 + ((struct ___Object___ *)parameter)->flag);
2934                         raw_test_pass(0xdd000000 + checkmask);
2935 #endif
2936                         if((((struct ___Object___ *)parameter)->flag&andmask)==checkmask) {
2937                                 ismet = true;
2938                                 break;
2939                         }
2940                 }
2941                 if (!ismet) {
2942                         // flags are never suitable
2943                         // remove this obj from the queue
2944                         int next;
2945                         int UNUSED, UNUSED2;
2946                         int * enterflags;
2947 #ifdef RAWDEBUG
2948                         raw_test_pass(0xe996);
2949 #endif
2950                         ObjectHashget(pw->objectset, (int) parameter, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2951                         ObjectHashremove(pw->objectset, (int)parameter);
2952                         if (enterflags!=NULL)
2953                                 free(enterflags);
2954                         // release grabbed locks
2955                         for(j = 0; j < i; ++j) {
2956                                 releasewritelock(taskpointerarray[j+OFFSET]);
2957                         }
2958                         releasewritelock(parameter);
2959                         RUNFREE(currtpd->parameterArray);
2960                         RUNFREE(currtpd);
2961                         goto newtask;
2962                 }
2963         }
2964 #endif
2965       parameterpresent:
2966         ;
2967         /* Check that object still has necessary tags */
2968         for(j=0;j<pd->numbertags;j++) {
2969           int slotid=pd->tagarray[2*j]+numparams;
2970           struct ___TagDescriptor___ *tagd=currtpd->parameterArray[slotid];
2971           if (!containstag(parameter, tagd)) {
2972 #ifdef RAWDEBUG
2973                   raw_test_pass(0xe997);
2974 #endif
2975             RUNFREE(currtpd->parameterArray);
2976             RUNFREE(currtpd);
2977             goto newtask;
2978           }
2979         }
2980
2981         taskpointerarray[i+OFFSET]=parameter;
2982       }
2983       /* Copy the tags */
2984       for(;i<numtotal;i++) {
2985         taskpointerarray[i+OFFSET]=currtpd->parameterArray[i];
2986       }
2987
2988 #ifdef THREADSIMULATE
2989          for(i = 0; i < numparams; ++i) {
2990                   if(0 == isolateflags[i]) {
2991                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
2992                           if(tmpparam != tmpparam->original) {
2993                                   taskpointerarray[i+OFFSET] = tmpparam->original;
2994                           }
2995                   }
2996           }
2997 #endif
2998
2999       {
3000 #if 0
3001 #ifndef RAW
3002         /* Checkpoint the state */
3003         forward=allocateRuntimeHash(100);
3004         reverse=allocateRuntimeHash(100);
3005         //void ** checkpoint=makecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, forward, reverse);
3006 #endif
3007 #endif
3008         if (x=setjmp(error_handler)) {
3009           int counter;
3010           /* Recover */
3011 #ifndef RAW
3012 #ifdef DEBUG
3013           printf("Fatal Error=%d, Recovering!\n",x);
3014 #endif
3015 #endif
3016          /*
3017           genputtable(failedtasks,currtpd,currtpd);
3018           //restorecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, checkpoint, forward, reverse);
3019
3020           freeRuntimeHash(forward);
3021           freeRuntimeHash(reverse);
3022           freemalloc();
3023           forward=NULL;
3024           reverse=NULL;
3025           */
3026           //fflush(stdout);
3027 #ifdef RAW
3028 #ifdef RAWDEBUG
3029           raw_test_pass_reg(x);
3030 #endif
3031           raw_test_done(0xa009);
3032 #else
3033           exit(-1);
3034 #endif
3035         } else {
3036           /*if (injectfailures) {
3037             if ((((double)random())/RAND_MAX)<failurechance) {
3038               printf("\nINJECTING TASK FAILURE to %s\n", currtpd->task->name);
3039               longjmp(error_handler,10);
3040             }
3041           }*/
3042           /* Actually call task */
3043 #ifdef PRECISE_GC
3044           ((int *)taskpointerarray)[0]=currtpd->numParameters;
3045           taskpointerarray[1]=NULL;
3046 #endif
3047 execute:
3048           if(debugtask){
3049 #ifndef RAW
3050             printf("ENTER %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
3051 #endif
3052             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
3053 #ifndef RAW
3054             printf("EXIT %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
3055 #endif
3056           } else {
3057             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
3058           }
3059           if(lock) {
3060 #ifdef RAW
3061            for(i = 0; i < numparams; ++i) {
3062                   int j = 0;
3063                   struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
3064 #ifdef RAWDEBUG
3065                   raw_test_pass(0xe998);
3066                 raw_test_pass(0xdd100000 + tmpparam->flag);
3067 #endif
3068                   releasewritelock(tmpparam);
3069           }
3070 #elif defined THREADSIMULATE
3071           for(i = 0; i < numparams; ++i) {
3072                   if(0 == isolateflags[i]) {
3073                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
3074                           releasewritelock(tmpparam);
3075                   }
3076           }
3077 #endif
3078           }
3079
3080 #if 0
3081 #ifndef RAW
3082           freeRuntimeHash(forward);
3083           freeRuntimeHash(reverse);
3084 #endif
3085 #endif
3086           freemalloc();
3087           // Free up task parameter descriptor
3088           RUNFREE(currtpd->parameterArray);
3089           RUNFREE(currtpd);
3090 #if 0
3091 #ifndef RAW
3092           forward=NULL;
3093           reverse=NULL;
3094 #endif
3095 #endif
3096         }
3097       }
3098     }
3099   }
3100 #ifdef RAWDEBUG
3101         raw_test_pass(0xe999);
3102 #endif
3103 }
3104
3105 /* This function processes an objects tags */
3106 void processtags(struct parameterdescriptor *pd, int index, struct parameterwrapper *parameter, int * iteratorcount, int *statusarray, int numparams) {
3107   int i;
3108
3109   for(i=0;i<pd->numbertags;i++) {
3110     int slotid=pd->tagarray[2*i];
3111     int tagid=pd->tagarray[2*i+1];
3112
3113     if (statusarray[slotid+numparams]==0) {
3114       parameter->iterators[*iteratorcount].istag=1;
3115       parameter->iterators[*iteratorcount].tagid=tagid;
3116       parameter->iterators[*iteratorcount].slot=slotid+numparams;
3117       parameter->iterators[*iteratorcount].tagobjectslot=index;
3118       statusarray[slotid+numparams]=1;
3119       (*iteratorcount)++;
3120     }
3121   }
3122 }
3123
3124
3125 void processobject(struct parameterwrapper *parameter, int index, struct parameterdescriptor *pd, int *iteratorcount, int * statusarray, int numparams) {
3126   int i;
3127   int tagcount=0;
3128   struct ObjectHash * objectset=((struct parameterwrapper *)pd->queue)->objectset;
3129
3130   parameter->iterators[*iteratorcount].istag=0;
3131   parameter->iterators[*iteratorcount].slot=index;
3132   parameter->iterators[*iteratorcount].objectset=objectset;
3133   statusarray[index]=1;
3134
3135   for(i=0;i<pd->numbertags;i++) {
3136     int slotid=pd->tagarray[2*i];
3137     int tagid=pd->tagarray[2*i+1];
3138     if (statusarray[slotid+numparams]!=0) {
3139       /* This tag has already been enqueued, use it to narrow search */
3140       parameter->iterators[*iteratorcount].tagbindings[tagcount]=slotid+numparams;
3141       tagcount++;
3142     }
3143   }
3144   parameter->iterators[*iteratorcount].numtags=tagcount;
3145
3146   (*iteratorcount)++;
3147 }
3148
3149 /* This function builds the iterators for a task & parameter */
3150
3151 void builditerators(struct taskdescriptor * task, int index, struct parameterwrapper * parameter) {
3152   int statusarray[MAXTASKPARAMS];
3153   int i;
3154   int numparams=task->numParameters;
3155   int iteratorcount=0;
3156   for(i=0;i<MAXTASKPARAMS;i++) statusarray[i]=0;
3157
3158   statusarray[index]=1; /* Initial parameter */
3159   /* Process tags for initial iterator */
3160
3161   processtags(task->descriptorarray[index], index, parameter, & iteratorcount, statusarray, numparams);
3162
3163   while(1) {
3164   loopstart:
3165     /* Check for objects with existing tags */
3166     for(i=0;i<numparams;i++) {
3167       if (statusarray[i]==0) {
3168         struct parameterdescriptor *pd=task->descriptorarray[i];
3169         int j;
3170         for(j=0;j<pd->numbertags;j++) {
3171           int slotid=pd->tagarray[2*j];
3172           if(statusarray[slotid+numparams]!=0) {
3173             processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3174             processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3175             goto loopstart;
3176           }
3177         }
3178       }
3179     }
3180
3181     /* Next do objects w/ unbound tags*/
3182
3183     for(i=0;i<numparams;i++) {
3184       if (statusarray[i]==0) {
3185         struct parameterdescriptor *pd=task->descriptorarray[i];
3186         if (pd->numbertags>0) {
3187           processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3188           processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3189           goto loopstart;
3190         }
3191       }
3192     }
3193
3194     /* Nothing with a tag enqueued */
3195
3196     for(i=0;i<numparams;i++) {
3197       if (statusarray[i]==0) {
3198         struct parameterdescriptor *pd=task->descriptorarray[i];
3199         processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
3200         processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
3201         goto loopstart;
3202       }
3203     }
3204
3205     /* Nothing left */
3206     return;
3207   }
3208 }
3209
3210  void printdebug() {
3211    int i;
3212    int j;
3213 #ifdef THREADSIMULATE
3214    int numofcore = pthread_getspecific(key);
3215    for(i=0;i<numtasks[numofcore];i++) {
3216            struct taskdescriptor * task=taskarray[numofcore][i];
3217 #else
3218 #ifdef RAW
3219            if(corenum > NUMCORES - 1) {
3220                    return;
3221            }
3222 #endif
3223    for(i=0;i<numtasks[corenum];i++) {
3224      struct taskdescriptor * task=taskarray[corenum][i];
3225 #endif
3226 #ifndef RAW
3227      printf("%s\n", task->name);
3228 #endif
3229      for(j=0;j<task->numParameters;j++) {
3230        struct parameterdescriptor *param=task->descriptorarray[j];
3231        struct parameterwrapper *parameter=param->queue;
3232        struct ObjectHash * set=parameter->objectset;
3233        struct ObjectIterator objit;
3234 #ifndef RAW
3235        printf("  Parameter %d\n", j);
3236 #endif
3237        ObjectHashiterator(set, &objit);
3238        while(ObjhasNext(&objit)) {
3239          struct ___Object___ * obj=(struct ___Object___ *)Objkey(&objit);
3240          struct ___Object___ * tagptr=obj->___tags___;
3241          int nonfailed=Objdata4(&objit);
3242          int numflags=Objdata3(&objit);
3243          int flags=Objdata2(&objit);
3244          Objnext(&objit);
3245 #ifndef RAW
3246          printf("    Contains %lx\n", obj);
3247          printf("      flag=%d\n", obj->flag);
3248 #endif
3249          if (tagptr==NULL) {
3250          } else if (tagptr->type==TAGTYPE) {
3251 #ifndef RAW
3252            printf("      tag=%lx\n",tagptr);
3253 #endif
3254            ;
3255          } else {
3256            int tagindex=0;
3257            struct ArrayObject *ao=(struct ArrayObject *)tagptr;
3258            for(;tagindex<ao->___cachedCode___;tagindex++) {
3259 #ifndef RAW
3260              printf("      tag=%lx\n",ARRAYGET(ao, struct ___TagDescriptor___*, tagindex));
3261 #endif
3262            }
3263          }
3264        }
3265      }
3266    }
3267  }
3268
3269
3270 /* This function processes the task information to create queues for
3271    each parameter type. */
3272
3273 void processtasks() {
3274   int i;
3275 #ifdef RAW
3276           if(corenum > NUMCORES - 1) {
3277                   return;
3278           }
3279 #endif
3280 #ifdef THREADSIMULATE
3281   int numofcore = pthread_getspecific(key);
3282   for(i=0;i<numtasks[numofcore];i++) {
3283           struct taskdescriptor *task=taskarray[numofcore][i];
3284 #else
3285   for(i=0;i<numtasks[corenum];i++) {
3286     struct taskdescriptor * task=taskarray[corenum][i];
3287 #endif
3288     int j;
3289
3290         /* Build objectsets */
3291         for(j=0;j<task->numParameters;j++) {
3292       struct parameterdescriptor *param=task->descriptorarray[j];
3293       struct parameterwrapper *parameter=param->queue;
3294           parameter->objectset=allocateObjectHash(10);
3295           parameter->task=task;
3296     }
3297
3298     /* Build iterators for parameters */
3299     for(j=0;j<task->numParameters;j++) {
3300       struct parameterdescriptor *param=task->descriptorarray[j];
3301       struct parameterwrapper *parameter=param->queue;
3302       builditerators(task, j, parameter);
3303     }
3304  }
3305 }
3306
3307 void toiReset(struct tagobjectiterator * it) {
3308   if (it->istag) {
3309     it->tagobjindex=0;
3310   } else if (it->numtags>0) {
3311     it->tagobjindex=0;
3312   } else {
3313     ObjectHashiterator(it->objectset, &it->it);
3314   }
3315 }
3316
3317 int toiHasNext(struct tagobjectiterator *it, void ** objectarray OPTARG(int * failed)) {
3318   if (it->istag) {
3319     /* Iterate tag */
3320     /* Get object with tags */
3321     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3322     struct ___Object___ *tagptr=obj->___tags___;
3323     if (tagptr->type==TAGTYPE) {
3324       if ((it->tagobjindex==0)&& /* First object */
3325           (it->tagid==((struct ___TagDescriptor___ *)tagptr)->flag)) /* Right tag type */
3326         return 1;
3327       else
3328         return 0;
3329     } else {
3330       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3331       int tagindex=it->tagobjindex;
3332       for(;tagindex<ao->___cachedCode___;tagindex++) {
3333         struct ___TagDescriptor___ *td=ARRAYGET(ao, struct ___TagDescriptor___ *, tagindex);
3334         if (td->flag==it->tagid) {
3335           it->tagobjindex=tagindex; /* Found right type of tag */
3336           return 1;
3337         }
3338       }
3339       return 0;
3340     }
3341   } else if (it->numtags>0) {
3342     /* Use tags to locate appropriate objects */
3343     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3344     struct ___Object___ *objptr=tag->flagptr;
3345     int i;
3346     if (objptr->type!=OBJECTARRAYTYPE) {
3347       if (it->tagobjindex>0)
3348         return 0;
3349       if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3350         return 0;
3351       for(i=1;i<it->numtags;i++) {
3352         struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3353         if (!containstag(objptr,tag2))
3354           return 0;
3355       }
3356       return 1;
3357     } else {
3358       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3359       int tagindex;
3360       int i;
3361       for(tagindex=it->tagobjindex;tagindex<ao->___cachedCode___;tagindex++) {
3362         struct ___Object___ *objptr=ARRAYGET(ao, struct ___Object___*, tagindex);
3363         if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3364           continue;
3365         for(i=1;i<it->numtags;i++) {
3366           struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3367           if (!containstag(objptr,tag2))
3368             goto nexttag;
3369         }
3370         it->tagobjindex=tagindex;
3371         return 1;
3372       nexttag:
3373         ;
3374       }
3375       it->tagobjindex=tagindex;
3376       return 0;
3377     }
3378   } else {
3379     return ObjhasNext(&it->it);
3380   }
3381 }
3382
3383 int containstag(struct ___Object___ *ptr, struct ___TagDescriptor___ *tag) {
3384   int j;
3385   struct ___Object___ * objptr=tag->flagptr;
3386   if (objptr->type==OBJECTARRAYTYPE) {
3387     struct ArrayObject *ao=(struct ArrayObject *)objptr;
3388     for(j=0;j<ao->___cachedCode___;j++) {
3389       if (ptr==ARRAYGET(ao, struct ___Object___*, j))
3390         return 1;
3391     }
3392     return 0;
3393   } else
3394     return objptr==ptr;
3395 }
3396
3397 void toiNext(struct tagobjectiterator *it , void ** objectarray OPTARG(int * failed)) {
3398   /* hasNext has all of the intelligence */
3399   if(it->istag) {
3400     /* Iterate tag */
3401     /* Get object with tags */
3402     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3403     struct ___Object___ *tagptr=obj->___tags___;
3404     if (tagptr->type==TAGTYPE) {
3405       it->tagobjindex++;
3406       objectarray[it->slot]=tagptr;
3407     } else {
3408       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3409       objectarray[it->slot]=ARRAYGET(ao, struct ___TagDescriptor___ *, it->tagobjindex++);
3410     }
3411   } else if (it->numtags>0) {
3412     /* Use tags to locate appropriate objects */
3413     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3414     struct ___Object___ *objptr=tag->flagptr;
3415     if (objptr->type!=OBJECTARRAYTYPE) {
3416       it->tagobjindex++;
3417       objectarray[it->slot]=objptr;
3418     } else {
3419       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3420       objectarray[it->slot]=ARRAYGET(ao, struct ___Object___ *, it->tagobjindex++);
3421     }
3422   } else {
3423     /* Iterate object */
3424     objectarray[it->slot]=(void *)Objkey(&it->it);
3425     Objnext(&it->it);
3426   }
3427 }
3428 #endif