2 #include "multicoregccompact.h"
3 #include "runtime_arch.h"
4 #include "multicoreruntime.h"
5 #include "multicoregarbage.h"
7 INLINE bool gc_checkCoreStatus() {
8 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
9 for(int i = 0; i < NUMCORES4GC; ++i) {
10 if(gccorestatus[i] != 0) {
11 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
15 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
19 INLINE void gc_resetCoreStatus() {
20 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
21 for(int i = 0; i < NUMCORES4GC; ++i) {
24 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
27 // should be invoked with interrupt closed
28 INLINE int assignSpareMem_I(unsigned int sourcecore,unsigned int * requiredmem,unsigned int * tomove,unsigned int * startaddr) {
30 BLOCKINDEX(gcloads[sourcecore], b);
31 unsigned int boundptr = BOUNDPTR(b);
32 unsigned int remain = boundptr - gcloads[sourcecore];
33 unsigned int memneed = requiredmem + BAMBOO_CACHE_LINE_SIZE;
34 *startaddr = gcloads[sourcecore];
35 *tomove = gcfilledblocks[sourcecore] + 1;
36 if(memneed < remain) {
37 gcloads[sourcecore] += memneed;
40 // next available block
41 gcfilledblocks[sourcecore] += 1;
42 unsigned int newbase = 0;
43 BASEPTR(sourcecore, gcfilledblocks[sourcecore], &newbase);
44 gcloads[sourcecore] = newbase;
45 return requiredmem-remain;
49 INLINE int assignSpareMem(unsigned int sourcecore,unsigned int * requiredmem,unsigned int * tomove,unsigned int * startaddr) {
50 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
51 int retval=assignSpareMem_I(sourcecore, requiredmem, tomove, startaddr);
52 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
56 INLINE void compact2Heaptophelper_I(unsigned int coren,unsigned int* p,unsigned int* numblocks,unsigned int* remain) {
58 unsigned int memneed = gcrequiredmems[coren] + BAMBOO_CACHE_LINE_SIZE;
59 if(STARTUPCORE == coren) {
62 gcdstcore = gctopcore;
63 gcblock2fill = *numblocks + 1;
65 if(BAMBOO_CHECK_SEND_MODE()) {
66 cache_msg_4_I(coren,GCMOVESTART,gctopcore,*p,(*numblocks)+1);
68 send_msg_4_I(coren,GCMOVESTART,gctopcore,*p,(*numblocks)+1);
71 if(memneed < *remain) {
73 gcrequiredmems[coren] = 0;
74 gcloads[gctopcore] += memneed;
75 *remain = *remain - memneed;
77 // next available block
79 gcfilledblocks[gctopcore] += 1;
80 unsigned int newbase = 0;
81 BASEPTR(gctopcore, gcfilledblocks[gctopcore], &newbase);
82 gcloads[gctopcore] = newbase;
83 gcrequiredmems[coren] -= *remain - BAMBOO_CACHE_LINE_SIZE;
84 gcstopblock[gctopcore]++;
85 gctopcore = NEXTTOPCORE(gctopblock);
87 *numblocks = gcstopblock[gctopcore];
88 *p = gcloads[gctopcore];
90 *remain=GC_BLOCK_REMAIN_SIZE(b, (*p));
95 INLINE void compact2Heaptop() {
96 // no cores with spare mem and some cores are blocked with pending move
97 // find the current heap top and make them move to the heap top
99 unsigned int numblocks = gcfilledblocks[gctopcore];
100 p = gcloads[gctopcore];
103 unsigned int remain=GC_BLOCK_REMAIN_SIZE(b, p);
104 // check if the top core finishes
105 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
106 if(gccorestatus[gctopcore] != 0) {
107 // let the top core finishes its own work first
108 compact2Heaptophelper_I(gctopcore, &p, &numblocks, &remain);
109 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
112 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
114 for(int i = 0; i < NUMCORES4GC; i++) {
115 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
116 if((gccorestatus[i] != 0) && (gcrequiredmems[i] > 0)) {
117 compact2Heaptophelper_I(i, &p, &numblocks, &remain);
118 if(gccorestatus[gctopcore] != 0) {
119 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
120 // the top core is not free now
124 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
128 INLINE void resolvePendingMoveRequest() {
131 bool nosparemem = true;
132 bool haspending = false;
133 bool hasrunning = false;
134 bool noblock = false;
135 unsigned int dstcore = 0; // the core who need spare mem
136 unsigned int sourcecore = 0; // the core who has spare mem
137 for(i = j = 0; (i < NUMCORES4GC) && (j < NUMCORES4GC); ) {
139 // check if there are cores with spare mem
140 if(gccorestatus[i] == 0) {
141 // finished working, check if it still have spare mem
142 if(gcfilledblocks[i] < gcstopblock[i]) {
143 // still have spare mem
151 if(gccorestatus[j] != 0) {
152 // not finished, check if it has pending move requests
153 if((gcfilledblocks[j]==gcstopblock[j])&&(gcrequiredmems[j]>0)) {
162 if(!nosparemem && haspending) {
164 unsigned int tomove = 0;
165 unsigned int startaddr = 0;
166 gcrequiredmems[dstcore] = assignSpareMem(sourcecore,gcrequiredmems[dstcore],&tomove,&startaddr);
167 if(STARTUPCORE == dstcore) {
168 gcdstcore = sourcecore;
170 gcmovestartaddr = startaddr;
171 gcblock2fill = tomove;
173 send_msg_4(dstcore,GCMOVESTART,sourcecore,startaddr,tomove);
182 if(!hasrunning && !noblock) {
183 gc_status_info.gcphase = SUBTLECOMPACTPHASE;
188 // If out of boundary of valid shared memory, return false, else return true
189 INLINE bool nextSBlock(struct moveHelper * orig) {
190 orig->blockbase = orig->blockbound;
192 bool sbchanged = false;
193 unsigned int origptr = orig->ptr;
194 unsigned int blockbase = orig->blockbase;
195 unsigned int blockbound = orig->blockbound;
196 unsigned int bound = orig->bound;
198 // check if across a big block
199 // TODO now do not zero out the whole memory, maybe the last two conditions
201 if((blockbase>=bound)||(origptr>=bound)||((origptr!=NULL)&&(*((int*)origptr))==0)||((*((int*)blockbase))==0)) {
203 // end of current heap block, jump to next one
205 BASEPTR(BAMBOO_NUM_OF_CORE, orig->numblocks, &(orig->base));
206 if(orig->base >= gcbaseva + BAMBOO_SHARED_MEM_SIZE) {
208 orig->ptr = orig->base; // set current ptr to out of boundary too
211 orig->blockbase = orig->base;
212 orig->sblockindex=(unsigned int)(orig->blockbase-gcbaseva)/BAMBOO_SMEM_SIZE;
214 unsigned int blocknum = 0;
215 BLOCKINDEX(orig->base, blocknum);
216 if(bamboo_smemtbl[blocknum] == 0) {
218 goto innernextSBlock;
220 // check the bamboo_smemtbl to decide the real bound
221 orig->bound = orig->base + bamboo_smemtbl[blocknum];
222 } else if(0 == (orig->blockbase%BAMBOO_SMEM_SIZE)) {
223 orig->sblockindex += 1;
227 // check if this sblock should be skipped or have special start point
228 int sbstart = gcsbstarttbl[orig->sblockindex];
231 orig->sblockindex += 1;
232 orig->blockbase += BAMBOO_SMEM_SIZE;
233 goto outernextSBlock;
234 } else if((sbstart != 0) && (sbchanged)) {
235 // the first time to access this SBlock
236 // not start from the very beginning
237 orig->blockbase = sbstart;
240 // setup information for this sblock
241 orig->blockbound = orig->blockbase+(unsigned int)*((int*)(orig->blockbase));
242 orig->offset = BAMBOO_CACHE_LINE_SIZE;
243 orig->ptr = orig->blockbase + orig->offset;
244 if(orig->ptr >= orig->bound) {
245 // met a lobj, move to next block
246 goto innernextSBlock;
252 // return false if there are no available data to compact
253 INLINE bool initOrig_Dst(struct moveHelper * orig,struct moveHelper * to) {
256 to->top = to->offset = BAMBOO_CACHE_LINE_SIZE;
257 to->bound = BAMBOO_SMEM_SIZE_L;
258 BASEPTR(BAMBOO_NUM_OF_CORE, to->numblocks, &(to->base));
260 unsigned int tobase = to->base;
261 to->ptr = tobase + to->offset;
266 unsigned int blocknum = 0;
267 BLOCKINDEX(orig->base, blocknum);
268 unsigned int origbase = orig->base;
269 // check the bamboo_smemtbl to decide the real bound
270 orig->bound = origbase + (unsigned int)bamboo_smemtbl[blocknum];
271 orig->blockbase = origbase;
272 orig->sblockindex = (unsigned int)(origbase - gcbaseva) / BAMBOO_SMEM_SIZE;
274 int sbstart = gcsbstarttbl[orig->sblockindex];
277 orig->blockbound=gcbaseva+BAMBOO_SMEM_SIZE*(orig->sblockindex+1);
278 return nextSBlock(orig);
279 } else if(sbstart != 0) {
280 orig->blockbase = sbstart;
282 orig->blockbound = orig->blockbase + *((int*)(orig->blockbase));
283 orig->offset = BAMBOO_CACHE_LINE_SIZE;
284 orig->ptr = orig->blockbase + orig->offset;
289 INLINE void nextBlock(struct moveHelper * to) {
290 to->top = to->bound + BAMBOO_CACHE_LINE_SIZE; // header!
291 to->bound += BAMBOO_SMEM_SIZE;
293 BASEPTR(BAMBOO_NUM_OF_CORE, to->numblocks, &(to->base));
294 to->offset = BAMBOO_CACHE_LINE_SIZE;
295 to->ptr = to->base + to->offset;
298 INLINE unsigned int findValidObj(struct moveHelper * orig,struct moveHelper * to,int * type) {
299 unsigned int size = 0;
301 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, to->ptr, false);
302 unsigned int origptr = (unsigned int)(orig->ptr);
303 unsigned int origbound = (unsigned int)orig->bound;
304 unsigned int origblockbound = (unsigned int)orig->blockbound;
305 if((origptr >= origbound) || (origptr == origblockbound)) {
306 if(!nextSBlock(orig)) {
307 // finished, no more data
312 // check the obj's type, size and mark flag
313 *type = ((int *)(origptr))[0];
316 // end of this block, go to next one
317 if(!nextSBlock(orig)) {
318 // finished, no more data
322 } else if(*type < NUMCLASSES) {
324 size = classsize[*type];
327 struct ArrayObject *ao=(struct ArrayObject *)(origptr);
328 unsigned int elementsize=classsize[*type];
329 unsigned int length=ao->___length___;
330 size=(unsigned int)sizeof(struct ArrayObject)+(unsigned int)(length*elementsize);
336 // endaddr does not contain spaces for headers
337 INLINE bool moveobj(struct moveHelper * orig, struct moveHelper * to, unsigned int stopblock) {
343 unsigned int size = findValidObj(orig, to, &type);
344 unsigned int isize = 0;
347 // finished, no more data
350 ALIGNSIZE(size, &isize); // no matter is the obj marked or not
351 // should be able to across
352 void * origptr = orig->ptr;
354 GETMARKED(markedstatus, origptr);
356 if(markedstatus==MARKEDFIRST) {
357 unsigned int totop = (unsigned int)to->top;
358 unsigned int tobound = (unsigned int)to->bound;
359 BAMBOO_ASSERT(totop<=tobound);
360 GCPROFILE_RECORD_LIVE_OBJ();
361 // marked obj, copy it to current heap top
362 // check to see if remaining space is enough
363 if((unsigned int)(totop + isize) > tobound) {
364 // fill 0 indicating the end of this block
365 BAMBOO_MEMSET_WH(to->ptr, '\0', tobound - totop);
366 // fill the header of this block and then go to next block
367 to->offset += tobound - totop;
368 CLOSEBLOCK(to->base, to->offset);
369 #ifdef GC_CACHE_ADAPT
370 unsigned int tmp_ptr = to->ptr;
373 if((to->top+isize)>(to->bound)) tprintf("%x, %x, %d, %d, %d, %d \n", to->ptr, orig->ptr, to->top, to->bound, isize, size);
374 BAMBOO_ASSERT((to->top+isize)<=(to->bound));
375 #ifdef GC_CACHE_ADAPT
376 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, tmp_ptr, true);
378 if(stopblock == to->numblocks) {
379 // already fulfilled the block
383 BAMBOO_ASSERT((to->top+isize)<=(to->bound));
384 // set the mark field to 2, indicating that this obj has been moved
385 // and need to be flushed
386 unsigned int toptr = (unsigned int)to->ptr;
387 if(toptr != origptr) {
388 if((unsigned int)(origptr) < (unsigned int)(toptr+size)) {
389 memmove(toptr, origptr, size);
391 memcpy(toptr, origptr, size);
393 // fill the remaining space with -2
394 BAMBOO_MEMSET_WH((unsigned int)(toptr+size), -2, isize-size);
396 // store mapping info
397 gcmappingtbl[OBJMAPPINGINDEX((unsigned int)origptr)]=(unsigned int)toptr;
398 gccurr_heaptop -= isize;
402 BAMBOO_ASSERT((to->top)<=(to->bound));
403 #ifdef GC_CACHE_ADAPT
404 unsigned int tmp_ptr = to->ptr;
405 #endif // GC_CACHE_ADAPT
406 if(to->top == to->bound) {
407 CLOSEBLOCK(to->base, to->offset);
410 #ifdef GC_CACHE_ADAPT
411 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, tmp_ptr, true);
418 return ((((unsigned int)(orig->ptr) > (unsigned int)(orig->bound))||((unsigned int)(orig->ptr) == (unsigned int)(orig->blockbound)))&&!nextSBlock(orig));
421 // should be invoked with interrupt closed
422 bool gcfindSpareMem_I(unsigned int * startaddr,unsigned int * tomove,unsigned int * dstcore,unsigned int requiredmem,unsigned int requiredcore) {
423 for(int k = 0; k < NUMCORES4GC; k++) {
424 if((gccorestatus[k] == 0) && (gcfilledblocks[k] < gcstopblock[k])) {
425 // check if this stopped core has enough mem
426 assignSpareMem_I(k, requiredmem, tomove, startaddr);
431 // if can not find spare mem right now, hold the request
432 gcrequiredmems[requiredcore] = requiredmem;
437 bool gcfindSpareMem(unsigned int * startaddr,unsigned int * tomove,unsigned int * dstcore,unsigned int requiredmem,unsigned int requiredcore) {
438 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
439 bool retval=gcfindSpareMem_I(startaddr, tomove, dstcore, requiredmem, requiredcore);
440 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
444 bool compacthelper(struct moveHelper * orig,struct moveHelper * to,int * filledblocks,unsigned int * heaptopptr,bool * localcompact, bool lbmove) {
445 bool loadbalancemove = lbmove;
446 // scan over all objs in this block, compact the marked objs
447 // loop stop when finishing either scanning all active objs or
448 // fulfilled the gcstopblock
450 while((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
451 if(moveobj(orig, to, gcblock2fill)) {
455 CACHEADAPT_SAMPLING_DATA_CONVERT(to->ptr);
456 // if no objs have been compact, do nothing,
457 // otherwise, fill the header of this block
458 if(to->offset > (unsigned int)BAMBOO_CACHE_LINE_SIZE) {
459 CLOSEBLOCK(to->base, to->offset);
463 to->top -= BAMBOO_CACHE_LINE_SIZE;
466 *heaptopptr = to->ptr;
467 *filledblocks = to->numblocks;
470 // send msgs to core coordinator indicating that the compact is finishing
471 // send compact finish message to core coordinator
472 if(STARTUPCORE == BAMBOO_NUM_OF_CORE) {
473 gcfilledblocks[BAMBOO_NUM_OF_CORE] = *filledblocks;
474 gcloads[BAMBOO_NUM_OF_CORE] = *heaptopptr;
475 //tprintf("--finish compact: %d, %d, %d, %x, %x \n", BAMBOO_NUM_OF_CORE, loadbalancemove, *filledblocks, *heaptopptr, gccurr_heaptop);
476 if((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
479 if(gcfindSpareMem(&gcmovestartaddr,&gcblock2fill,&gcdstcore,gccurr_heaptop,BAMBOO_NUM_OF_CORE)) {
485 gccorestatus[BAMBOO_NUM_OF_CORE] = 0;
487 // write back to the Main Memory and release any DTLB entry for the
488 // last block as someone else might later write into it
489 // flush the shared heap
490 //BAMBOO_CACHE_FLUSH_L2();
494 if((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
497 send_msg_6(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,loadbalancemove,*filledblocks,*heaptopptr,gccurr_heaptop);
500 send_msg_6(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,loadbalancemove,*filledblocks,*heaptopptr, 0);
501 // write back to the Main Memory and release any DTLB entry for the
502 // last block as someone else might later write into it.
503 // flush the shared heap
507 if(orig->ptr < gcmarkedptrbound) {
508 // still have unpacked obj
511 loadbalancemove = true;
514 to->ptr = gcmovestartaddr;
515 to->numblocks = gcblock2fill - 1;
516 to->bound = BLOCKBOUND(to->numblocks);
517 BASEPTR(gcdstcore, to->numblocks, &(to->base));
518 to->offset = to->ptr - to->base;
519 to->top=(to->numblocks==0)?(to->offset):(to->bound-BAMBOO_SMEM_SIZE+to->offset);
521 to->offset = BAMBOO_CACHE_LINE_SIZE;
522 to->ptr += to->offset; // for header
523 to->top += to->offset;
524 *localcompact = (gcdstcore == BAMBOO_NUM_OF_CORE);
525 CACHEADAPT_SAMPLING_DATA_REVISE_INIT(orig, to);
532 BAMBOO_ASSERT(COMPACTPHASE == gc_status_info.gcphase);
535 // initialize pointers for comapcting
536 struct moveHelper * orig = (struct moveHelper *)RUNMALLOC(sizeof(struct moveHelper));
537 struct moveHelper * to = (struct moveHelper *)RUNMALLOC(sizeof(struct moveHelper));
538 if(!initOrig_Dst(orig, to)) {
539 // no available data to compact
540 // send compact finish msg to STARTUP core
541 send_msg_6(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,false,0,to->base,0);
545 CACHEADAPT_SAMPLING_DATA_REVISE_INIT(orig, to);
547 unsigned int filledblocks = 0;
548 unsigned int heaptopptr = 0;
549 bool localcompact = true;
550 compacthelper(orig, to, &filledblocks, &heaptopptr, &localcompact, false);
556 void compact_master(struct moveHelper * orig, struct moveHelper * to) {
557 // initialize pointers for comapcting
558 initOrig_Dst(orig, to);
559 CACHEADAPT_SAMPLING_DATA_REVISE_INIT(orig, to);
560 int filledblocks = 0;
561 unsigned int heaptopptr = 0;
562 bool finishcompact = false;
563 bool iscontinue = true;
564 bool localcompact = true;
566 while((COMPACTPHASE == gc_status_info.gcphase) || (SUBTLECOMPACTPHASE == gc_status_info.gcphase)) {
567 if((!finishcompact) && iscontinue) {
568 finishcompact = compacthelper(orig,to,&filledblocks,&heaptopptr,&localcompact, lbmove);
571 if(gc_checkCoreStatus()) {
572 // all cores have finished compacting restore the gcstatus of all cores
573 gc_resetCoreStatus();
576 // check if there are spare mem for pending move requires
577 if(COMPACTPHASE == gc_status_info.gcphase) {
578 resolvePendingMoveRequest();
586 to->ptr = gcmovestartaddr;
587 to->numblocks = gcblock2fill - 1;
588 to->bound = BLOCKBOUND(to->numblocks);
589 BASEPTR(gcdstcore, to->numblocks, &(to->base));
590 to->offset = to->ptr - to->base;
591 to->top = (to->numblocks==0)?(to->offset):(to->bound-BAMBOO_SMEM_SIZE+to->offset);
593 to->offset = BAMBOO_CACHE_LINE_SIZE;
594 to->ptr += to->offset; // for header
595 to->top += to->offset;
596 localcompact = (gcdstcore == BAMBOO_NUM_OF_CORE);
600 } else if(!finishcompact) {
608 #endif // MULTICORE_GC