b9c346104662474aa8f9098acc83750641039bf8
[iot2.git] / benchmarks / other / PhoneInterface / Control / app / src / main / java / iotcloud / Table.java
1 package iotcloud;
2
3 import java.util.Iterator;
4 import java.util.Random;
5 import java.util.Arrays;
6 import java.util.Map;
7 import java.util.Set;
8 import java.util.List;
9 import java.util.Vector;
10 import java.util.HashMap;
11 import java.util.HashSet;
12 import java.util.ArrayList;
13 import java.util.Collections;
14 import java.nio.ByteBuffer;
15 import android.content.*;
16
17 /**
18  * IoTTable data structure.  Provides client interface.
19  * @author Brian Demsky
20  * @version 1.0
21  */
22
23 final public class Table {
24
25         /* Constants */
26         static final int FREE_SLOTS = 10; // Number of slots that should be kept free
27         static final int SKIP_THRESHOLD = 10;
28         static final double RESIZE_MULTIPLE = 1.2;
29         static final double RESIZE_THRESHOLD = 0.75;
30         static final int REJECTED_THRESHOLD = 5;
31
32         /* Helper Objects */
33         private SlotBuffer buffer = null;
34         private CloudComm cloud = null;
35         private Random random = null;
36         private TableStatus liveTableStatus = null;
37         private PendingTransaction pendingTransactionBuilder = null; // Pending Transaction used in building a Pending Transaction
38         private Transaction lastPendingTransactionSpeculatedOn = null; // Last transaction that was speculated on from the pending transaction
39         private Transaction firstPendingTransaction = null; // first transaction in the pending transaction list
40
41         /* Variables */
42         private int numberOfSlots = 0;  // Number of slots stored in buffer
43         private int bufferResizeThreshold = 0; // Threshold on the number of live slots before a resize is needed
44         private long liveSlotCount = 0; // Number of currently live slots
45         private long oldestLiveSlotSequenceNumver = 0;  // Smallest sequence number of the slot with a live entry
46         private long localMachineId = 0; // Machine ID of this client device
47         private long sequenceNumber = 0; // Largest sequence number a client has received
48         // private int smallestTableStatusSeen = -1; // Smallest Table Status that was seen in the latest slots sent from the server
49         // private int largestTableStatusSeen = -1; // Largest Table Status that was seen in the latest slots sent from the server
50         private long localTransactionSequenceNumber = 0; // Local sequence number counter for transactions
51         private long lastTransactionSequenceNumberSpeculatedOn = -1; // the last transaction that was speculated on
52         private long oldestTransactionSequenceNumberSpeculatedOn = -1; // the oldest transaction that was speculated on
53         private long localArbitrationSequenceNumber = 0;
54         private boolean hadPartialSendToServer = false;
55         private boolean attemptedToSendToServer = false;
56         private long expectedsize;
57         private boolean didFindTableStatus = false;
58         private long currMaxSize = 0;
59
60         private Slot lastSlotAttemptedToSend = null;
61         private boolean lastIsNewKey = false;
62         private int lastNewSize = 0;
63         private Map<Transaction, List<Integer>> lastTransactionPartsSent = null;
64         private List<Entry> lastPendingSendArbitrationEntriesToDelete = null;
65         private NewKey lastNewKey = null;
66
67
68         /* Data Structures  */
69         private Map<IoTString, KeyValue> committedKeyValueTable = null; // Table of committed key value pairs
70         private Map<IoTString, KeyValue> speculatedKeyValueTable = null; // Table of speculated key value pairs, if there is a speculative value
71         private Map<IoTString, KeyValue> pendingTransactionSpeculatedKeyValueTable = null; // Table of speculated key value pairs, if there is a speculative value from the pending transactions
72         private Map<IoTString, NewKey> liveNewKeyTable = null; // Table of live new keys
73         private HashMap<Long, Pair<Long, Liveness>> lastMessageTable = null; // Last message sent by a client machine id -> (Seq Num, Slot or LastMessage);
74         private HashMap<Long, HashSet<RejectedMessage>> rejectedMessageWatchListTable = null; // Table of machine Ids and the set of rejected messages they have not seen yet
75         private Map<IoTString, Long> arbitratorTable = null; // Table of keys and their arbitrators
76         private Map<Pair<Long, Long>, Abort> liveAbortTable = null; // Table live abort messages
77         private Map<Long, Map<Pair<Long, Integer>, TransactionPart>> newTransactionParts = null; // transaction parts that are seen in this latest round of slots from the server
78         private Map<Long, Map<Pair<Long, Integer>, CommitPart>> newCommitParts = null; // commit parts that are seen in this latest round of slots from the server
79         private Map<Long, Long> lastArbitratedTransactionNumberByArbitratorTable = null; // Last transaction sequence number that an arbitrator arbitrated on
80         private Map<Long, Transaction> liveTransactionBySequenceNumberTable = null; // live transaction grouped by the sequence number
81         private Map<Pair<Long, Long>, Transaction> liveTransactionByTransactionIdTable = null; // live transaction grouped by the transaction ID
82         private Map<Long, Map<Long, Commit>> liveCommitsTable = null;
83         private Map<IoTString, Commit> liveCommitsByKeyTable = null;
84         private Map<Long, Long> lastCommitSeenSequenceNumberByArbitratorTable = null;
85         private Vector<Long> rejectedSlotList = null; // List of rejected slots that have yet to be sent to the server
86         private List<Transaction> pendingTransactionQueue = null;
87         private List<ArbitrationRound> pendingSendArbitrationRounds = null;
88         private List<Entry> pendingSendArbitrationEntriesToDelete = null;
89         private Map<Transaction, List<Integer>> transactionPartsSent = null;
90         private Map<Long, TransactionStatus> outstandingTransactionStatus = null;
91         private Map<Long, Abort> liveAbortsGeneratedByLocal = null;
92         private Set<Pair<Long, Long>> offlineTransactionsCommittedAndAtServer = null;
93         private Map<Long, Pair<String, Integer>> localCommunicationTable = null;
94         private Map<Long, Long> lastTransactionSeenFromMachineFromServer = null;
95         private Map<Long, Long> lastArbitrationDataLocalSequenceNumberSeenFromArbitrator = null;
96
97
98         public Table(String baseurl, String password, long _localMachineId, int listeningPort, Context context) {
99                 localMachineId = _localMachineId;
100                 cloud = new CloudComm(this, baseurl, password, listeningPort, context);
101
102                 init();
103         }
104
105         public Table(CloudComm _cloud, long _localMachineId) {
106                 localMachineId = _localMachineId;
107                 cloud = _cloud;
108
109                 init();
110         }
111
112         /**
113          * Init all the stuff needed for for table usage
114          */
115         private void init() {
116
117                 // Init helper objects
118                 random = new Random();
119                 buffer = new SlotBuffer();
120
121                 // Set Variables
122                 oldestLiveSlotSequenceNumver = 1;
123
124                 // init data structs
125                 committedKeyValueTable = new HashMap<IoTString, KeyValue>();
126                 speculatedKeyValueTable = new HashMap<IoTString, KeyValue>();
127                 pendingTransactionSpeculatedKeyValueTable = new HashMap<IoTString, KeyValue>();
128                 liveNewKeyTable = new HashMap<IoTString, NewKey>();
129                 lastMessageTable = new HashMap<Long, Pair<Long, Liveness>>();
130                 rejectedMessageWatchListTable = new HashMap<Long, HashSet<RejectedMessage>>();
131                 arbitratorTable = new HashMap<IoTString, Long>();
132                 liveAbortTable = new HashMap<Pair<Long, Long>, Abort>();
133                 newTransactionParts = new HashMap<Long, Map<Pair<Long, Integer>, TransactionPart>>();
134                 newCommitParts = new HashMap<Long, Map<Pair<Long, Integer>, CommitPart>>();
135                 lastArbitratedTransactionNumberByArbitratorTable = new HashMap<Long, Long>();
136                 liveTransactionBySequenceNumberTable = new HashMap<Long, Transaction>();
137                 liveTransactionByTransactionIdTable = new HashMap<Pair<Long, Long>, Transaction>();
138                 liveCommitsTable = new HashMap<Long, Map<Long, Commit>>();
139                 liveCommitsByKeyTable = new HashMap<IoTString, Commit>();
140                 lastCommitSeenSequenceNumberByArbitratorTable = new HashMap<Long, Long>();
141                 rejectedSlotList = new Vector<Long>();
142                 pendingTransactionQueue = new ArrayList<Transaction>();
143                 pendingSendArbitrationEntriesToDelete = new ArrayList<Entry>();
144                 transactionPartsSent = new HashMap<Transaction, List<Integer>>();
145                 outstandingTransactionStatus = new HashMap<Long, TransactionStatus>();
146                 liveAbortsGeneratedByLocal = new HashMap<Long, Abort>();
147                 offlineTransactionsCommittedAndAtServer = new HashSet<Pair<Long, Long>>();
148                 localCommunicationTable = new HashMap<Long, Pair<String, Integer>>();
149                 lastTransactionSeenFromMachineFromServer = new HashMap<Long, Long>();
150                 pendingSendArbitrationRounds = new ArrayList<ArbitrationRound>();
151                 lastArbitrationDataLocalSequenceNumberSeenFromArbitrator = new HashMap<Long, Long>();
152
153
154                 // Other init stuff
155                 numberOfSlots = buffer.capacity();
156                 setResizeThreshold();
157         }
158
159         // TODO: delete method
160         public synchronized void printSlots() {
161                 long o = buffer.getOldestSeqNum();
162                 long n = buffer.getNewestSeqNum();
163
164                 int[] types = new int[10];
165
166                 int num = 0;
167
168                 int livec = 0;
169                 int deadc = 0;
170                 for (long i = o; i < (n + 1); i++) {
171                         Slot s = buffer.getSlot(i);
172
173                         Vector<Entry> entries = s.getEntries();
174
175                         for (Entry e : entries) {
176                                 if (e.isLive()) {
177                                         int type = e.getType();
178                                         types[type] = types[type] + 1;
179                                         num++;
180                                         livec++;
181                                 } else {
182                                         deadc++;
183                                 }
184                         }
185                 }
186
187                 for (int i = 0; i < 10; i++) {
188                         System.out.println(i + "    " + types[i]);
189                 }
190                 System.out.println("Live count:   " + livec);
191                 System.out.println("Dead count:   " + deadc);
192                 System.out.println("Old:   " + o);
193                 System.out.println("New:   " + n);
194                 System.out.println("Size:   " + buffer.size());
195                 // System.out.println("Commits:   " + liveCommitsTable.size());
196                 System.out.println("pendingTrans:   " + pendingTransactionQueue.size());
197                 System.out.println("Trans Status Out:   " + outstandingTransactionStatus.size());
198
199                 for (Long k : lastArbitratedTransactionNumberByArbitratorTable.keySet()) {
200                         System.out.println(k + ": " + lastArbitratedTransactionNumberByArbitratorTable.get(k));
201                 }
202
203
204                 for (Long a : liveCommitsTable.keySet()) {
205                         for (Long b : liveCommitsTable.get(a).keySet()) {
206                                 for (KeyValue kv : liveCommitsTable.get(a).get(b).getKeyValueUpdateSet()) {
207                                         System.out.print(kv + " ");
208                                 }
209                                 System.out.print("|| ");
210                         }
211                         System.out.println();
212                 }
213
214         }
215
216         /**
217          * Initialize the table by inserting a table status as the first entry into the table status
218          * also initialize the crypto stuff.
219          */
220         public synchronized void initTable() throws ServerException {
221                 cloud.initSecurity();
222
223                 // Create the first insertion into the block chain which is the table status
224                 Slot s = new Slot(this, 1, localMachineId);
225                 TableStatus status = new TableStatus(s, numberOfSlots);
226                 s.addEntry(status);
227                 Slot[] array = cloud.putSlot(s, numberOfSlots);
228
229                 if (array == null) {
230                         array = new Slot[] {s};
231                         // update local block chain
232                         validateAndUpdate(array, true);
233                 } else if (array.length == 1) {
234                         // in case we did push the slot BUT we failed to init it
235                         validateAndUpdate(array, true);
236                 } else {
237                         throw new Error("Error on initialization");
238                 }
239         }
240
241         /**
242          * Rebuild the table from scratch by pulling the latest block chain from the server.
243          */
244         public synchronized void rebuild() throws ServerException {
245                 // Just pull the latest slots from the server
246                 Slot[] newslots = cloud.getSlots(sequenceNumber + 1);
247                 validateAndUpdate(newslots, true);
248         }
249
250         // public String toString() {
251         //      String retString = " Committed Table: \n";
252         //      retString += "---------------------------\n";
253         //      retString += commitedTable.toString();
254
255         //      retString += "\n\n";
256
257         //      retString += " Speculative Table: \n";
258         //      retString += "---------------------------\n";
259         //      retString += speculativeTable.toString();
260
261         //      return retString;
262         // }
263
264         public synchronized void addLocalCommunication(long arbitrator, String hostName, int portNumber) {
265                 localCommunicationTable.put(arbitrator, new Pair<String, Integer>(hostName, portNumber));
266         }
267
268         public synchronized Long getArbitrator(IoTString key) {
269                 return arbitratorTable.get(key);
270         }
271
272         public synchronized void close() {
273                 cloud.close();
274         }
275
276         public synchronized IoTString getCommitted(IoTString key)  {
277                 KeyValue kv = committedKeyValueTable.get(key);
278
279                 if (kv != null) {
280                         return kv.getValue();
281                 } else {
282                         return null;
283                 }
284         }
285
286         public synchronized IoTString getSpeculative(IoTString key) {
287                 KeyValue kv = pendingTransactionSpeculatedKeyValueTable.get(key);
288
289                 if (kv == null) {
290                         kv = speculatedKeyValueTable.get(key);
291                 }
292
293                 if (kv == null) {
294                         kv = committedKeyValueTable.get(key);
295                 }
296
297                 if (kv != null) {
298                         return kv.getValue();
299                 } else {
300                         return null;
301                 }
302         }
303
304         public synchronized IoTString getCommittedAtomic(IoTString key) {
305                 KeyValue kv = committedKeyValueTable.get(key);
306
307                 if (arbitratorTable.get(key) == null) {
308                         throw new Error("Key not Found.");
309                 }
310
311                 // Make sure new key value pair matches the current arbitrator
312                 if (!pendingTransactionBuilder.checkArbitrator(arbitratorTable.get(key))) {
313                         // TODO: Maybe not throw en error
314                         throw new Error("Not all Key Values Match Arbitrator.");
315                 }
316
317                 if (kv != null) {
318                         pendingTransactionBuilder.addKVGuard(new KeyValue(key, kv.getValue()));
319                         return kv.getValue();
320                 } else {
321                         pendingTransactionBuilder.addKVGuard(new KeyValue(key, null));
322                         return null;
323                 }
324         }
325
326         public synchronized IoTString getSpeculativeAtomic(IoTString key) {
327                 if (arbitratorTable.get(key) == null) {
328                         throw new Error("Key not Found.");
329                 }
330
331                 // Make sure new key value pair matches the current arbitrator
332                 if (!pendingTransactionBuilder.checkArbitrator(arbitratorTable.get(key))) {
333                         // TODO: Maybe not throw en error
334                         throw new Error("Not all Key Values Match Arbitrator.");
335                 }
336
337                 KeyValue kv = pendingTransactionSpeculatedKeyValueTable.get(key);
338
339                 if (kv == null) {
340                         kv = speculatedKeyValueTable.get(key);
341                 }
342
343                 if (kv == null) {
344                         kv = committedKeyValueTable.get(key);
345                 }
346
347                 if (kv != null) {
348                         pendingTransactionBuilder.addKVGuard(new KeyValue(key, kv.getValue()));
349                         return kv.getValue();
350                 } else {
351                         pendingTransactionBuilder.addKVGuard(new KeyValue(key, null));
352                         return null;
353                 }
354         }
355
356         public synchronized boolean update()  {
357                 try {
358                         Slot[] newSlots = cloud.getSlots(sequenceNumber + 1);
359                         validateAndUpdate(newSlots, false);
360                         sendToServer(null);
361
362
363                         updateLiveTransactionsAndStatus();
364
365                         return true;
366                 } catch (Exception e) {
367                         // e.printStackTrace();
368
369                         for (Long m : localCommunicationTable.keySet()) {
370                                 updateFromLocal(m);
371                         }
372                 }
373
374                 return false;
375         }
376
377         public synchronized boolean createNewKey(IoTString keyName, long machineId) throws ServerException {
378                 while (true) {
379                         if (arbitratorTable.get(keyName) != null) {
380                                 // There is already an arbitrator
381                                 return false;
382                         }
383
384                         NewKey newKey = new NewKey(null, keyName, machineId);
385                         if (sendToServer(newKey)) {
386                                 // If successfully inserted
387                                 return true;
388                         }
389                 }
390         }
391
392         public synchronized void startTransaction() {
393                 // Create a new transaction, invalidates any old pending transactions.
394                 pendingTransactionBuilder = new PendingTransaction(localMachineId);
395         }
396
397         public synchronized void addKV(IoTString key, IoTString value) {
398
399                 // Make sure it is a valid key
400                 if (arbitratorTable.get(key) == null) {
401                         throw new Error("Key not Found.");
402                 }
403
404                 // Make sure new key value pair matches the current arbitrator
405                 if (!pendingTransactionBuilder.checkArbitrator(arbitratorTable.get(key))) {
406                         // TODO: Maybe not throw en error
407                         throw new Error("Not all Key Values Match Arbitrator.");
408                 }
409
410                 // Add the key value to this transaction
411                 KeyValue kv = new KeyValue(key, value);
412                 pendingTransactionBuilder.addKV(kv);
413         }
414
415         public synchronized TransactionStatus commitTransaction() {
416
417                 if (pendingTransactionBuilder.getKVUpdates().size() == 0) {
418                         // transaction with no updates will have no effect on the system
419                         return new TransactionStatus(TransactionStatus.StatusNoEffect, -1);
420                 }
421
422                 // Set the local transaction sequence number and increment
423                 pendingTransactionBuilder.setClientLocalSequenceNumber(localTransactionSequenceNumber);
424                 localTransactionSequenceNumber++;
425
426                 // Create the transaction status
427                 TransactionStatus transactionStatus = new TransactionStatus(TransactionStatus.StatusPending, pendingTransactionBuilder.getArbitrator());
428
429                 // Create the new transaction
430                 Transaction newTransaction = pendingTransactionBuilder.createTransaction();
431                 newTransaction.setTransactionStatus(transactionStatus);
432
433                 if (pendingTransactionBuilder.getArbitrator() != localMachineId) {
434                         // Add it to the queue and invalidate the builder for safety
435                         pendingTransactionQueue.add(newTransaction);
436                 } else {
437                         arbitrateOnLocalTransaction(newTransaction);
438                         updateLiveStateFromLocal();
439                 }
440
441                 pendingTransactionBuilder = new PendingTransaction(localMachineId);
442
443                 try {
444                         sendToServer(null);
445                 } catch (ServerException e) {
446
447                         Set<Long> arbitratorTriedAndFailed = new HashSet<Long>();
448                         for (Iterator<Transaction> iter = pendingTransactionQueue.iterator(); iter.hasNext(); ) {
449                                 Transaction transaction = iter.next();
450
451                                 if (arbitratorTriedAndFailed.contains(transaction.getArbitrator())) {
452                                         // Already contacted this client so ignore all attempts to contact this client
453                                         // to preserve ordering for arbitrator
454                                         continue;
455                                 }
456
457                                 Pair<Boolean, Boolean> sendReturn = sendTransactionToLocal(transaction);
458
459                                 if (sendReturn.getFirst()) {
460                                         // Failed to contact over local
461                                         arbitratorTriedAndFailed.add(transaction.getArbitrator());
462                                 } else {
463                                         // Successful contact or should not contact
464
465                                         if (sendReturn.getSecond()) {
466                                                 // did arbitrate
467                                                 iter.remove();
468                                         }
469                                 }
470                         }
471                 }
472
473                 updateLiveStateFromLocal();
474
475                 return transactionStatus;
476         }
477
478         /**
479          * Get the machine ID for this client
480          */
481         public long getMachineId() {
482                 return localMachineId;
483         }
484
485         /**
486          * Decrement the number of live slots that we currently have
487          */
488         public void decrementLiveCount() {
489                 liveSlotCount--;
490         }
491
492         /**
493          * Recalculate the new resize threshold
494          */
495         private void setResizeThreshold() {
496                 int resizeLower = (int) (RESIZE_THRESHOLD * numberOfSlots);
497                 bufferResizeThreshold = resizeLower - 1 + random.nextInt(numberOfSlots - resizeLower);
498         }
499
500
501         boolean lastInsertedNewKey = false;
502
503         private boolean sendToServer(NewKey newKey) throws ServerException {
504
505                 boolean fromRetry = false;
506
507                 try {
508                         if (hadPartialSendToServer) {
509                                 Slot[] newSlots = cloud.getSlots(sequenceNumber + 1);
510                                 if (newSlots.length == 0) {
511                                         fromRetry = true;
512                                         ThreeTuple<Boolean, Boolean, Slot[]> sendSlotsReturn = sendSlotsToServer(lastSlotAttemptedToSend, lastNewSize, lastIsNewKey);
513
514                                         if (sendSlotsReturn.getFirst()) {
515                                                 if (newKey != null) {
516                                                         if (lastInsertedNewKey && (lastNewKey.getKey() == newKey.getKey()) && (lastNewKey.getMachineID() == newKey.getMachineID())) {
517                                                                 newKey = null;
518                                                         }
519                                                 }
520
521                                                 for (Transaction transaction : lastTransactionPartsSent.keySet()) {
522                                                         transaction.resetServerFailure();
523
524                                                         // Update which transactions parts still need to be sent
525                                                         transaction.removeSentParts(lastTransactionPartsSent.get(transaction));
526
527                                                         // Add the transaction status to the outstanding list
528                                                         outstandingTransactionStatus.put(transaction.getSequenceNumber(), transaction.getTransactionStatus());
529
530                                                         // Update the transaction status
531                                                         transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentPartial);
532
533                                                         // Check if all the transaction parts were successfully sent and if so then remove it from pending
534                                                         if (transaction.didSendAllParts()) {
535                                                                 transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentFully);
536                                                                 pendingTransactionQueue.remove(transaction);
537                                                         }
538                                                 }
539                                         } else {
540
541                                                 newSlots = sendSlotsReturn.getThird();
542
543                                                 boolean isInserted = false;
544                                                 for (Slot s : newSlots) {
545                                                         if ((s.getSequenceNumber() == lastSlotAttemptedToSend.getSequenceNumber()) && (s.getMachineID() == localMachineId)) {
546                                                                 isInserted = true;
547                                                                 break;
548                                                         }
549                                                 }
550
551                                                 for (Slot s : newSlots) {
552                                                         if (isInserted) {
553                                                                 break;
554                                                         }
555
556                                                         // Process each entry in the slot
557                                                         for (Entry entry : s.getEntries()) {
558
559                                                                 if (entry.getType() == Entry.TypeLastMessage) {
560                                                                         LastMessage lastMessage = (LastMessage)entry;
561                                                                         if ((lastMessage.getMachineID() == localMachineId) && (lastMessage.getSequenceNumber() == lastSlotAttemptedToSend.getSequenceNumber())) {
562                                                                                 isInserted = true;
563                                                                                 break;
564                                                                         }
565                                                                 }
566                                                         }
567                                                 }
568
569                                                 if (isInserted) {
570                                                         if (newKey != null) {
571                                                                 if (lastInsertedNewKey && (lastNewKey.getKey() == newKey.getKey()) && (lastNewKey.getMachineID() == newKey.getMachineID())) {
572                                                                         newKey = null;
573                                                                 }
574                                                         }
575
576                                                         for (Transaction transaction : lastTransactionPartsSent.keySet()) {
577                                                                 transaction.resetServerFailure();
578
579                                                                 // Update which transactions parts still need to be sent
580                                                                 transaction.removeSentParts(lastTransactionPartsSent.get(transaction));
581
582                                                                 // Add the transaction status to the outstanding list
583                                                                 outstandingTransactionStatus.put(transaction.getSequenceNumber(), transaction.getTransactionStatus());
584
585                                                                 // Update the transaction status
586                                                                 transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentPartial);
587
588                                                                 // Check if all the transaction parts were successfully sent and if so then remove it from pending
589                                                                 if (transaction.didSendAllParts()) {
590                                                                         transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentFully);
591                                                                         pendingTransactionQueue.remove(transaction);
592                                                                 } else {
593                                                                         transaction.resetServerFailure();
594                                                                         // Set the transaction sequence number back to nothing
595                                                                         if (!transaction.didSendAPartToServer()) {
596                                                                                 transaction.setSequenceNumber(-1);
597                                                                         }
598                                                                 }
599                                                         }
600                                                 }
601                                         }
602
603                                         for (Transaction transaction : lastTransactionPartsSent.keySet()) {
604                                                 transaction.resetServerFailure();
605                                                 // Set the transaction sequence number back to nothing
606                                                 if (!transaction.didSendAPartToServer()) {
607                                                         transaction.setSequenceNumber(-1);
608                                                 }
609                                         }
610
611                                         if (sendSlotsReturn.getThird().length != 0) {
612                                                 // insert into the local block chain
613                                                 validateAndUpdate(sendSlotsReturn.getThird(), true);
614                                         }
615                                         // continue;
616                                 } else {
617                                         boolean isInserted = false;
618                                         for (Slot s : newSlots) {
619                                                 if ((s.getSequenceNumber() == lastSlotAttemptedToSend.getSequenceNumber()) && (s.getMachineID() == localMachineId)) {
620                                                         isInserted = true;
621                                                         break;
622                                                 }
623                                         }
624
625                                         for (Slot s : newSlots) {
626                                                 if (isInserted) {
627                                                         break;
628                                                 }
629
630                                                 // Process each entry in the slot
631                                                 for (Entry entry : s.getEntries()) {
632
633                                                         if (entry.getType() == Entry.TypeLastMessage) {
634                                                                 LastMessage lastMessage = (LastMessage)entry;
635                                                                 if ((lastMessage.getMachineID() == localMachineId) && (lastMessage.getSequenceNumber() == lastSlotAttemptedToSend.getSequenceNumber())) {
636                                                                         isInserted = true;
637                                                                         break;
638                                                                 }
639                                                         }
640                                                 }
641                                         }
642
643                                         if (isInserted) {
644                                                 if (newKey != null) {
645                                                         if (lastInsertedNewKey && (lastNewKey.getKey() == newKey.getKey()) && (lastNewKey.getMachineID() == newKey.getMachineID())) {
646                                                                 newKey = null;
647                                                         }
648                                                 }
649
650                                                 for (Transaction transaction : lastTransactionPartsSent.keySet()) {
651                                                         transaction.resetServerFailure();
652
653                                                         // Update which transactions parts still need to be sent
654                                                         transaction.removeSentParts(lastTransactionPartsSent.get(transaction));
655
656                                                         // Add the transaction status to the outstanding list
657                                                         outstandingTransactionStatus.put(transaction.getSequenceNumber(), transaction.getTransactionStatus());
658
659                                                         // Update the transaction status
660                                                         transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentPartial);
661
662                                                         // Check if all the transaction parts were successfully sent and if so then remove it from pending
663                                                         if (transaction.didSendAllParts()) {
664                                                                 transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentFully);
665                                                                 pendingTransactionQueue.remove(transaction);
666                                                         } else {
667                                                                 transaction.resetServerFailure();
668                                                                 // Set the transaction sequence number back to nothing
669                                                                 if (!transaction.didSendAPartToServer()) {
670                                                                         transaction.setSequenceNumber(-1);
671                                                                 }
672                                                         }
673                                                 }
674                                         } else {
675                                                 for (Transaction transaction : lastTransactionPartsSent.keySet()) {
676                                                         transaction.resetServerFailure();
677                                                         // Set the transaction sequence number back to nothing
678                                                         if (!transaction.didSendAPartToServer()) {
679                                                                 transaction.setSequenceNumber(-1);
680                                                         }
681                                                 }
682                                         }
683
684                                         // insert into the local block chain
685                                         validateAndUpdate(newSlots, true);
686                                 }
687                         }
688                 } catch (ServerException e) {
689                         throw e;
690                 }
691
692
693                 try {
694                         // While we have stuff that needs inserting into the block chain
695                         while ((pendingTransactionQueue.size() > 0) || (pendingSendArbitrationRounds.size() > 0) || (newKey != null)) {
696                                 fromRetry = false;
697
698                                 if (hadPartialSendToServer) {
699                                         throw new Error("Should Be error free");
700                                 }
701
702
703
704                                 // If there is a new key with same name then end
705                                 if ((newKey != null) && (arbitratorTable.get(newKey.getKey()) != null)) {
706                                         return false;
707                                 }
708
709                                 // Create the slot
710                                 Slot slot = new Slot(this, sequenceNumber + 1, localMachineId, buffer.getSlot(sequenceNumber).getHMAC());
711
712                                 // Try to fill the slot with data
713                                 ThreeTuple<Boolean, Integer, Boolean> fillSlotsReturn = fillSlot(slot, false, newKey);
714                                 boolean needsResize = fillSlotsReturn.getFirst();
715                                 int newSize = fillSlotsReturn.getSecond();
716                                 Boolean insertedNewKey = fillSlotsReturn.getThird();
717
718                                 if (needsResize) {
719                                         // Reset which transaction to send
720                                         for (Transaction transaction : transactionPartsSent.keySet()) {
721                                                 transaction.resetNextPartToSend();
722
723                                                 // Set the transaction sequence number back to nothing
724                                                 if (!transaction.didSendAPartToServer() && !transaction.getServerFailure()) {
725                                                         transaction.setSequenceNumber(-1);
726                                                 }
727                                         }
728
729                                         // Clear the sent data since we are trying again
730                                         pendingSendArbitrationEntriesToDelete.clear();
731                                         transactionPartsSent.clear();
732
733                                         // We needed a resize so try again
734                                         fillSlot(slot, true, newKey);
735                                 }
736
737                                 lastSlotAttemptedToSend = slot;
738                                 lastIsNewKey = (newKey != null);
739                                 lastInsertedNewKey = insertedNewKey;
740                                 lastNewSize = newSize;
741                                 lastNewKey = newKey;
742                                 lastTransactionPartsSent = new HashMap<Transaction, List<Integer>>(transactionPartsSent);
743                                 lastPendingSendArbitrationEntriesToDelete = new ArrayList<Entry>(pendingSendArbitrationEntriesToDelete);
744
745
746                                 ThreeTuple<Boolean, Boolean, Slot[]> sendSlotsReturn = sendSlotsToServer(slot, newSize, newKey != null);
747
748                                 if (sendSlotsReturn.getFirst()) {
749
750                                         // Did insert into the block chain
751
752                                         if (insertedNewKey) {
753                                                 // This slot was what was inserted not a previous slot
754
755                                                 // New Key was successfully inserted into the block chain so dont want to insert it again
756                                                 newKey = null;
757                                         }
758
759                                         // Remove the aborts and commit parts that were sent from the pending to send queue
760                                         for (Iterator<ArbitrationRound> iter = pendingSendArbitrationRounds.iterator(); iter.hasNext(); ) {
761                                                 ArbitrationRound round = iter.next();
762                                                 round.removeParts(pendingSendArbitrationEntriesToDelete);
763
764                                                 if (round.isDoneSending()) {
765                                                         // Sent all the parts
766                                                         iter.remove();
767                                                 }
768                                         }
769
770                                         for (Transaction transaction : transactionPartsSent.keySet()) {
771                                                 transaction.resetServerFailure();
772
773                                                 // Update which transactions parts still need to be sent
774                                                 transaction.removeSentParts(transactionPartsSent.get(transaction));
775
776                                                 // Add the transaction status to the outstanding list
777                                                 outstandingTransactionStatus.put(transaction.getSequenceNumber(), transaction.getTransactionStatus());
778
779                                                 // Update the transaction status
780                                                 transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentPartial);
781
782                                                 // Check if all the transaction parts were successfully sent and if so then remove it from pending
783                                                 if (transaction.didSendAllParts()) {
784                                                         transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentFully);
785                                                         pendingTransactionQueue.remove(transaction);
786                                                 }
787                                         }
788                                 } else {
789
790                                         // if (!sendSlotsReturn.getSecond()) {
791                                         //      for (Transaction transaction : lastTransactionPartsSent.keySet()) {
792                                         //              transaction.resetServerFailure();
793                                         //      }
794                                         // } else {
795                                         //      for (Transaction transaction : lastTransactionPartsSent.keySet()) {
796                                         //              transaction.resetServerFailure();
797
798                                         //              // Update which transactions parts still need to be sent
799                                         //              transaction.removeSentParts(transactionPartsSent.get(transaction));
800
801                                         //              // Add the transaction status to the outstanding list
802                                         //              outstandingTransactionStatus.put(transaction.getSequenceNumber(), transaction.getTransactionStatus());
803
804                                         //              // Update the transaction status
805                                         //              transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentPartial);
806
807                                         //              // Check if all the transaction parts were successfully sent and if so then remove it from pending
808                                         //              if (transaction.didSendAllParts()) {
809                                         //                      transaction.getTransactionStatus().setStatus(TransactionStatus.StatusSentFully);
810                                         //                      pendingTransactionQueue.remove(transaction);
811
812                                         //                      for (KeyValue kv : transaction.getKeyValueUpdateSet()) {
813                                         //                              System.out.println("Sent: " + kv + "  from: " + localMachineId + "   Slot:" + lastSlotAttemptedToSend.getSequenceNumber() + "  Claimed:" + transaction.getSequenceNumber());
814                                         //                      }
815                                         //              }
816                                         //      }
817                                         // }
818
819                                         // Reset which transaction to send
820                                         for (Transaction transaction : transactionPartsSent.keySet()) {
821                                                 transaction.resetNextPartToSend();
822                                                 // transaction.resetNextPartToSend();
823
824                                                 // Set the transaction sequence number back to nothing
825                                                 if (!transaction.didSendAPartToServer() && !transaction.getServerFailure()) {
826                                                         transaction.setSequenceNumber(-1);
827                                                 }
828                                         }
829                                 }
830
831                                 // Clear the sent data in preparation for next send
832                                 pendingSendArbitrationEntriesToDelete.clear();
833                                 transactionPartsSent.clear();
834
835                                 if (sendSlotsReturn.getThird().length != 0) {
836                                         // insert into the local block chain
837                                         validateAndUpdate(sendSlotsReturn.getThird(), true);
838                                 }
839                         }
840
841                 } catch (ServerException e) {
842
843                         if (e.getType() != ServerException.TypeInputTimeout) {
844                                 // e.printStackTrace();
845
846                                 // Nothing was able to be sent to the server so just clear these data structures
847                                 for (Transaction transaction : transactionPartsSent.keySet()) {
848                                         transaction.resetNextPartToSend();
849
850                                         // Set the transaction sequence number back to nothing
851                                         if (!transaction.didSendAPartToServer() && !transaction.getServerFailure()) {
852                                                 transaction.setSequenceNumber(-1);
853                                         }
854                                 }
855                         } else {
856                                 // There was a partial send to the server
857                                 hadPartialSendToServer = true;
858
859
860                                 // if (!fromRetry) {
861                                 //      lastTransactionPartsSent = new HashMap<Transaction, List<Integer>>(transactionPartsSent);
862                                 //      lastPendingSendArbitrationEntriesToDelete = new ArrayList<Entry>(pendingSendArbitrationEntriesToDelete);
863                                 // }
864
865                                 // Nothing was able to be sent to the server so just clear these data structures
866                                 for (Transaction transaction : transactionPartsSent.keySet()) {
867                                         transaction.resetNextPartToSend();
868                                         transaction.setServerFailure();
869                                 }
870                         }
871
872                         pendingSendArbitrationEntriesToDelete.clear();
873                         transactionPartsSent.clear();
874
875                         throw e;
876                 }
877
878                 return newKey == null;
879         }
880
881         private synchronized boolean updateFromLocal(long machineId) {
882                 Pair<String, Integer> localCommunicationInformation = localCommunicationTable.get(machineId);
883                 if (localCommunicationInformation == null) {
884                         // Cant talk to that device locally so do nothing
885                         return false;
886                 }
887
888                 // Get the size of the send data
889                 //int sendDataSize = Integer.BYTES + Long.BYTES;
890                 int sendDataSize = (Integer.SIZE + Long.SIZE)/8;
891
892                 Long lastArbitrationDataLocalSequenceNumber = (long) - 1;
893                 if (lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(machineId) != null) {
894                         lastArbitrationDataLocalSequenceNumber = lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(machineId);
895                 }
896
897                 byte[] sendData = new byte[sendDataSize];
898                 ByteBuffer bbEncode = ByteBuffer.wrap(sendData);
899
900                 // Encode the data
901                 bbEncode.putLong(lastArbitrationDataLocalSequenceNumber);
902                 bbEncode.putInt(0);
903
904                 // Send by local
905                 byte[] returnData = cloud.sendLocalData(sendData, localCommunicationInformation.getFirst(), localCommunicationInformation.getSecond());
906
907                 if (returnData == null) {
908                         // Could not contact server
909                         return false;
910                 }
911
912                 // Decode the data
913                 ByteBuffer bbDecode = ByteBuffer.wrap(returnData);
914                 int numberOfEntries = bbDecode.getInt();
915
916                 for (int i = 0; i < numberOfEntries; i++) {
917                         byte type = bbDecode.get();
918                         if (type == Entry.TypeAbort) {
919                                 Abort abort = (Abort)Abort.decode(null, bbDecode);
920                                 processEntry(abort);
921                         } else if (type == Entry.TypeCommitPart) {
922                                 CommitPart commitPart = (CommitPart)CommitPart.decode(null, bbDecode);
923                                 processEntry(commitPart);
924                         }
925                 }
926
927                 updateLiveStateFromLocal();
928
929                 return true;
930         }
931
932         private Pair<Boolean, Boolean> sendTransactionToLocal(Transaction transaction) {
933
934                 // Get the devices local communications
935                 Pair<String, Integer> localCommunicationInformation = localCommunicationTable.get(transaction.getArbitrator());
936
937                 if (localCommunicationInformation == null) {
938                         // Cant talk to that device locally so do nothing
939                         return new Pair<Boolean, Boolean>(true, false);
940                 }
941
942                 // Get the size of the send data
943                 //int sendDataSize = Integer.BYTES + Long.BYTES;
944                 int sendDataSize = (Integer.SIZE + Long.SIZE)/8;
945                 for (TransactionPart part : transaction.getParts().values()) {
946                         sendDataSize += part.getSize();
947                 }
948
949                 Long lastArbitrationDataLocalSequenceNumber = (long) - 1;
950                 if (lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(transaction.getArbitrator()) != null) {
951                         lastArbitrationDataLocalSequenceNumber = lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(transaction.getArbitrator());
952                 }
953
954                 // Make the send data size
955                 byte[] sendData = new byte[sendDataSize];
956                 ByteBuffer bbEncode = ByteBuffer.wrap(sendData);
957
958                 // Encode the data
959                 bbEncode.putLong(lastArbitrationDataLocalSequenceNumber);
960                 bbEncode.putInt(transaction.getParts().size());
961                 for (TransactionPart part : transaction.getParts().values()) {
962                         part.encode(bbEncode);
963                 }
964
965
966                 // Send by local
967                 byte[] returnData = cloud.sendLocalData(sendData, localCommunicationInformation.getFirst(), localCommunicationInformation.getSecond());
968
969                 if (returnData == null) {
970                         // Could not contact server
971                         return new Pair<Boolean, Boolean>(true, false);
972                 }
973
974                 // Decode the data
975                 ByteBuffer bbDecode = ByteBuffer.wrap(returnData);
976                 boolean didCommit = bbDecode.get() == 1;
977                 boolean couldArbitrate = bbDecode.get() == 1;
978                 int numberOfEntries = bbDecode.getInt();
979                 boolean foundAbort = false;
980
981                 for (int i = 0; i < numberOfEntries; i++) {
982                         byte type = bbDecode.get();
983                         if (type == Entry.TypeAbort) {
984                                 Abort abort = (Abort)Abort.decode(null, bbDecode);
985
986                                 if ((abort.getTransactionMachineId() == localMachineId) && (abort.getTransactionClientLocalSequenceNumber() == transaction.getClientLocalSequenceNumber())) {
987                                         foundAbort = true;
988                                 }
989
990                                 processEntry(abort);
991                         } else if (type == Entry.TypeCommitPart) {
992                                 CommitPart commitPart = (CommitPart)CommitPart.decode(null, bbDecode);
993                                 processEntry(commitPart);
994                         }
995                 }
996
997                 updateLiveStateFromLocal();
998
999                 if (couldArbitrate) {
1000                         TransactionStatus status =  transaction.getTransactionStatus();
1001                         if (didCommit) {
1002                                 status.setStatus(TransactionStatus.StatusCommitted);
1003                         } else {
1004                                 status.setStatus(TransactionStatus.StatusAborted);
1005                         }
1006                 } else {
1007                         TransactionStatus status =  transaction.getTransactionStatus();
1008                         if (foundAbort) {
1009                                 status.setStatus(TransactionStatus.StatusAborted);
1010                         } else {
1011                                 status.setStatus(TransactionStatus.StatusCommitted);
1012                         }
1013                 }
1014
1015                 return new Pair<Boolean, Boolean>(false, true);
1016         }
1017
1018         public synchronized byte[] acceptDataFromLocal(byte[] data) {
1019
1020                 // Decode the data
1021                 ByteBuffer bbDecode = ByteBuffer.wrap(data);
1022                 long lastArbitratedSequenceNumberSeen = bbDecode.getLong();
1023                 int numberOfParts = bbDecode.getInt();
1024
1025                 // If we did commit a transaction or not
1026                 boolean didCommit = false;
1027                 boolean couldArbitrate = false;
1028
1029                 if (numberOfParts != 0) {
1030
1031                         // decode the transaction
1032                         Transaction transaction = new Transaction();
1033                         for (int i = 0; i < numberOfParts; i++) {
1034                                 bbDecode.get();
1035                                 TransactionPart newPart = (TransactionPart)TransactionPart.decode(null, bbDecode);
1036                                 transaction.addPartDecode(newPart);
1037                         }
1038
1039                         // Arbitrate on transaction and pull relevant return data
1040                         Pair<Boolean, Boolean> localArbitrateReturn = arbitrateOnLocalTransaction(transaction);
1041                         couldArbitrate = localArbitrateReturn.getFirst();
1042                         didCommit = localArbitrateReturn.getSecond();
1043
1044                         updateLiveStateFromLocal();
1045
1046                         // Transaction was sent to the server so keep track of it to prevent double commit
1047                         if (transaction.getSequenceNumber() != -1) {
1048                                 offlineTransactionsCommittedAndAtServer.add(transaction.getId());
1049                         }
1050                 }
1051
1052                 // The data to send back
1053                 int returnDataSize = 0;
1054                 List<Entry> unseenArbitrations = new ArrayList<Entry>();
1055
1056                 // Get the aborts to send back
1057                 List<Long> abortLocalSequenceNumbers = new ArrayList<Long >(liveAbortsGeneratedByLocal.keySet());
1058                 Collections.sort(abortLocalSequenceNumbers);
1059                 for (Long localSequenceNumber : abortLocalSequenceNumbers) {
1060                         if (localSequenceNumber <= lastArbitratedSequenceNumberSeen) {
1061                                 continue;
1062                         }
1063
1064                         Abort abort = liveAbortsGeneratedByLocal.get(localSequenceNumber);
1065                         unseenArbitrations.add(abort);
1066                         returnDataSize += abort.getSize();
1067                 }
1068
1069                 // Get the commits to send back
1070                 Map<Long, Commit> commitForClientTable = liveCommitsTable.get(localMachineId);
1071                 if (commitForClientTable != null) {
1072                         List<Long> commitLocalSequenceNumbers = new ArrayList<Long>(commitForClientTable.keySet());
1073                         Collections.sort(commitLocalSequenceNumbers);
1074
1075                         for (Long localSequenceNumber : commitLocalSequenceNumbers) {
1076                                 Commit commit = commitForClientTable.get(localSequenceNumber);
1077
1078                                 if (localSequenceNumber <= lastArbitratedSequenceNumberSeen) {
1079                                         continue;
1080                                 }
1081
1082                                 unseenArbitrations.addAll(commit.getParts().values());
1083
1084                                 for (CommitPart commitPart : commit.getParts().values()) {
1085                                         returnDataSize += commitPart.getSize();
1086                                 }
1087                         }
1088                 }
1089
1090                 // Number of arbitration entries to decode
1091                 //returnDataSize += 2 * Integer.BYTES;
1092                 returnDataSize += 2 * Integer.SIZE/8;
1093
1094                 // Boolean of did commit or not
1095                 if (numberOfParts != 0) {
1096                         //returnDataSize += Byte.BYTES;
1097                         returnDataSize += Byte.SIZE/8;
1098                 }
1099
1100                 // Data to send Back
1101                 byte[] returnData = new byte[returnDataSize];
1102                 ByteBuffer bbEncode = ByteBuffer.wrap(returnData);
1103
1104                 if (numberOfParts != 0) {
1105                         if (didCommit) {
1106                                 bbEncode.put((byte)1);
1107                         } else {
1108                                 bbEncode.put((byte)0);
1109                         }
1110                         if (couldArbitrate) {
1111                                 bbEncode.put((byte)1);
1112                         } else {
1113                                 bbEncode.put((byte)0);
1114                         }
1115                 }
1116
1117                 bbEncode.putInt(unseenArbitrations.size());
1118                 for (Entry entry : unseenArbitrations) {
1119                         entry.encode(bbEncode);
1120                 }
1121
1122                 return returnData;
1123         }
1124
1125         private ThreeTuple<Boolean, Boolean, Slot[]> sendSlotsToServer(Slot slot, int newSize, boolean isNewKey)  throws ServerException {
1126
1127                 boolean attemptedToSendToServerTmp = attemptedToSendToServer;
1128                 attemptedToSendToServer = true;
1129
1130                 boolean inserted = false;
1131                 boolean lastTryInserted = false;
1132
1133                 Slot[] array = cloud.putSlot(slot, newSize);
1134                 if (array == null) {
1135                         array = new Slot[] {slot};
1136                         rejectedSlotList.clear();
1137                         inserted = true;
1138                 }       else {
1139                         if (array.length == 0) {
1140                                 throw new Error("Server Error: Did not send any slots");
1141                         }
1142
1143                         // if (attemptedToSendToServerTmp) {
1144                         if (hadPartialSendToServer) {
1145
1146                                 boolean isInserted = false;
1147                                 for (Slot s : array) {
1148                                         if ((s.getSequenceNumber() == slot.getSequenceNumber()) && (s.getMachineID() == localMachineId)) {
1149                                                 isInserted = true;
1150                                                 break;
1151                                         }
1152                                 }
1153
1154                                 for (Slot s : array) {
1155                                         if (isInserted) {
1156                                                 break;
1157                                         }
1158
1159                                         // Process each entry in the slot
1160                                         for (Entry entry : s.getEntries()) {
1161
1162                                                 if (entry.getType() == Entry.TypeLastMessage) {
1163                                                         LastMessage lastMessage = (LastMessage)entry;
1164
1165                                                         if ((lastMessage.getMachineID() == localMachineId) && (lastMessage.getSequenceNumber() == slot.getSequenceNumber())) {
1166                                                                 isInserted = true;
1167                                                                 break;
1168                                                         }
1169                                                 }
1170                                         }
1171                                 }
1172
1173                                 if (!isInserted) {
1174                                         rejectedSlotList.add(slot.getSequenceNumber());
1175                                         lastTryInserted = false;
1176                                 } else {
1177                                         lastTryInserted = true;
1178                                 }
1179                         } else {
1180                                 rejectedSlotList.add(slot.getSequenceNumber());
1181                                 lastTryInserted = false;
1182                         }
1183                 }
1184
1185                 return new ThreeTuple<Boolean, Boolean, Slot[]>(inserted, lastTryInserted, array);
1186         }
1187
1188         /**
1189          * Returns false if a resize was needed
1190          */
1191         private ThreeTuple<Boolean, Integer, Boolean> fillSlot(Slot slot, boolean resize, NewKey newKeyEntry) {
1192                 int newSize = 0;
1193                 if (liveSlotCount > bufferResizeThreshold) {
1194                         resize = true; //Resize is forced
1195                 }
1196
1197                 if (resize) {
1198                         newSize = (int) (numberOfSlots * RESIZE_MULTIPLE);
1199                         TableStatus status = new TableStatus(slot, newSize);
1200                         slot.addEntry(status);
1201                 }
1202
1203                 // Fill with rejected slots first before doing anything else
1204                 doRejectedMessages(slot);
1205
1206                 // Do mandatory rescue of entries
1207                 ThreeTuple<Boolean, Boolean, Long> mandatoryRescueReturn = doMandatoryResuce(slot, resize);
1208
1209                 // Extract working variables
1210                 boolean needsResize = mandatoryRescueReturn.getFirst();
1211                 boolean seenLiveSlot = mandatoryRescueReturn.getSecond();
1212                 long currentRescueSequenceNumber = mandatoryRescueReturn.getThird();
1213
1214                 if (needsResize && !resize) {
1215                         // We need to resize but we are not resizing so return false
1216                         return new ThreeTuple<Boolean, Integer, Boolean>(true, null, null);
1217                 }
1218
1219                 boolean inserted = false;
1220                 if (newKeyEntry != null) {
1221                         newKeyEntry.setSlot(slot);
1222                         if (slot.hasSpace(newKeyEntry)) {
1223                                 slot.addEntry(newKeyEntry);
1224                                 inserted = true;
1225                         }
1226                 }
1227
1228                 // Clear the transactions, aborts and commits that were sent previously
1229                 transactionPartsSent.clear();
1230                 pendingSendArbitrationEntriesToDelete.clear();
1231
1232                 for (ArbitrationRound round : pendingSendArbitrationRounds) {
1233                         boolean isFull = false;
1234                         round.generateParts();
1235                         List<Entry> parts = round.getParts();
1236
1237                         // Insert pending arbitration data
1238                         for (Entry arbitrationData : parts) {
1239
1240                                 // If it is an abort then we need to set some information
1241                                 if (arbitrationData instanceof Abort) {
1242                                         ((Abort)arbitrationData).setSequenceNumber(slot.getSequenceNumber());
1243                                 }
1244
1245                                 if (!slot.hasSpace(arbitrationData)) {
1246                                         // No space so cant do anything else with these data entries
1247                                         isFull = true;
1248                                         break;
1249                                 }
1250
1251                                 // Add to this current slot and add it to entries to delete
1252                                 slot.addEntry(arbitrationData);
1253                                 pendingSendArbitrationEntriesToDelete.add(arbitrationData);
1254                         }
1255
1256                         if (isFull) {
1257                                 break;
1258                         }
1259                 }
1260
1261                 if (pendingTransactionQueue.size() > 0) {
1262
1263                         Transaction transaction = pendingTransactionQueue.get(0);
1264
1265                         // Set the transaction sequence number if it has yet to be inserted into the block chain
1266                         // if ((!transaction.didSendAPartToServer() && !transaction.getServerFailure()) || (transaction.getSequenceNumber() == -1)) {
1267                         //      transaction.setSequenceNumber(slot.getSequenceNumber());
1268                         // }
1269
1270                         if ((!transaction.didSendAPartToServer()) || (transaction.getSequenceNumber() == -1)) {
1271                                 transaction.setSequenceNumber(slot.getSequenceNumber());
1272                         }
1273
1274
1275                         while (true) {
1276                                 TransactionPart part = transaction.getNextPartToSend();
1277
1278                                 if (part == null) {
1279                                         // Ran out of parts to send for this transaction so move on
1280                                         break;
1281                                 }
1282
1283                                 if (slot.hasSpace(part)) {
1284                                         slot.addEntry(part);
1285                                         List<Integer> partsSent = transactionPartsSent.get(transaction);
1286                                         if (partsSent == null) {
1287                                                 partsSent = new ArrayList<Integer>();
1288                                                 transactionPartsSent.put(transaction, partsSent);
1289                                         }
1290                                         partsSent.add(part.getPartNumber());
1291                                         transactionPartsSent.put(transaction, partsSent);
1292                                 } else {
1293                                         break;
1294                                 }
1295                         }
1296                 }
1297
1298                 // Fill the remainder of the slot with rescue data
1299                 doOptionalRescue(slot, seenLiveSlot, currentRescueSequenceNumber, resize);
1300
1301                 return new ThreeTuple<Boolean, Integer, Boolean>(false, newSize, inserted);
1302         }
1303
1304         private void doRejectedMessages(Slot s) {
1305                 if (! rejectedSlotList.isEmpty()) {
1306                         /* TODO: We should avoid generating a rejected message entry if
1307                          * there is already a sufficient entry in the queue (e.g.,
1308                          * equalsto value of true and same sequence number).  */
1309
1310                         long old_seqn = rejectedSlotList.firstElement();
1311                         if (rejectedSlotList.size() > REJECTED_THRESHOLD) {
1312                                 long new_seqn = rejectedSlotList.lastElement();
1313                                 RejectedMessage rm = new RejectedMessage(s, s.getSequenceNumber(), localMachineId, old_seqn, new_seqn, false);
1314                                 s.addEntry(rm);
1315                         } else {
1316                                 long prev_seqn = -1;
1317                                 int i = 0;
1318                                 /* Go through list of missing messages */
1319                                 for (; i < rejectedSlotList.size(); i++) {
1320                                         long curr_seqn = rejectedSlotList.get(i);
1321                                         Slot s_msg = buffer.getSlot(curr_seqn);
1322                                         if (s_msg != null)
1323                                                 break;
1324                                         prev_seqn = curr_seqn;
1325                                 }
1326                                 /* Generate rejected message entry for missing messages */
1327                                 if (prev_seqn != -1) {
1328                                         RejectedMessage rm = new RejectedMessage(s, s.getSequenceNumber(), localMachineId, old_seqn, prev_seqn, false);
1329                                         s.addEntry(rm);
1330                                 }
1331                                 /* Generate rejected message entries for present messages */
1332                                 for (; i < rejectedSlotList.size(); i++) {
1333                                         long curr_seqn = rejectedSlotList.get(i);
1334                                         Slot s_msg = buffer.getSlot(curr_seqn);
1335                                         long machineid = s_msg.getMachineID();
1336                                         RejectedMessage rm = new RejectedMessage(s, s.getSequenceNumber(), machineid, curr_seqn, curr_seqn, true);
1337                                         s.addEntry(rm);
1338                                 }
1339                         }
1340                 }
1341         }
1342
1343         private ThreeTuple<Boolean, Boolean, Long> doMandatoryResuce(Slot slot, boolean resize) {
1344                 long newestSequenceNumber = buffer.getNewestSeqNum();
1345                 long oldestSequenceNumber = buffer.getOldestSeqNum();
1346                 if (oldestLiveSlotSequenceNumver < oldestSequenceNumber) {
1347                         oldestLiveSlotSequenceNumver = oldestSequenceNumber;
1348                 }
1349
1350                 long currentSequenceNumber = oldestLiveSlotSequenceNumver;
1351                 boolean seenLiveSlot = false;
1352                 long firstIfFull = newestSequenceNumber + 1 - numberOfSlots;    // smallest seq number in the buffer if it is full
1353                 long threshold = firstIfFull + FREE_SLOTS;      // we want the buffer to be clear of live entries up to this point
1354
1355
1356                 // Mandatory Rescue
1357                 for (; currentSequenceNumber < threshold; currentSequenceNumber++) {
1358                         Slot previousSlot = buffer.getSlot(currentSequenceNumber);
1359                         // Push slot number forward
1360                         if (! seenLiveSlot) {
1361                                 oldestLiveSlotSequenceNumver = currentSequenceNumber;
1362                         }
1363
1364                         if (!previousSlot.isLive()) {
1365                                 continue;
1366                         }
1367
1368                         // We have seen a live slot
1369                         seenLiveSlot = true;
1370
1371                         // Get all the live entries for a slot
1372                         Vector<Entry> liveEntries = previousSlot.getLiveEntries(resize);
1373
1374                         // Iterate over all the live entries and try to rescue them
1375                         for (Entry liveEntry : liveEntries) {
1376                                 if (slot.hasSpace(liveEntry)) {
1377
1378                                         // Enough space to rescue the entry
1379                                         slot.addEntry(liveEntry);
1380                                 } else if (currentSequenceNumber == firstIfFull) {
1381                                         //if there's no space but the entry is about to fall off the queue
1382                                         System.out.println("B"); //?
1383                                         return new ThreeTuple<Boolean, Boolean, Long>(true, seenLiveSlot, currentSequenceNumber);
1384
1385                                 }
1386                         }
1387                 }
1388
1389                 // Did not resize
1390                 return new ThreeTuple<Boolean, Boolean, Long>(false, seenLiveSlot, currentSequenceNumber);
1391         }
1392
1393         private void  doOptionalRescue(Slot s, boolean seenliveslot, long seqn, boolean resize) {
1394                 /* now go through live entries from least to greatest sequence number until
1395                  * either all live slots added, or the slot doesn't have enough room
1396                  * for SKIP_THRESHOLD consecutive entries*/
1397                 int skipcount = 0;
1398                 long newestseqnum = buffer.getNewestSeqNum();
1399                 search:
1400                 for (; seqn <= newestseqnum; seqn++) {
1401                         Slot prevslot = buffer.getSlot(seqn);
1402                         //Push slot number forward
1403                         if (!seenliveslot)
1404                                 oldestLiveSlotSequenceNumver = seqn;
1405
1406                         if (!prevslot.isLive())
1407                                 continue;
1408                         seenliveslot = true;
1409                         Vector<Entry> liveentries = prevslot.getLiveEntries(resize);
1410                         for (Entry liveentry : liveentries) {
1411                                 if (s.hasSpace(liveentry))
1412                                         s.addEntry(liveentry);
1413                                 else {
1414                                         skipcount++;
1415                                         if (skipcount > SKIP_THRESHOLD)
1416                                                 break search;
1417                                 }
1418                         }
1419                 }
1420         }
1421
1422         /**
1423          * Checks for malicious activity and updates the local copy of the block chain.
1424          */
1425         private void validateAndUpdate(Slot[] newSlots, boolean acceptUpdatesToLocal) {
1426
1427                 // The cloud communication layer has checked slot HMACs already before decoding
1428                 if (newSlots.length == 0) {
1429                         return;
1430                 }
1431
1432                 // Make sure all slots are newer than the last largest slot this client has seen
1433                 long firstSeqNum = newSlots[0].getSequenceNumber();
1434                 if (firstSeqNum <= sequenceNumber) {
1435                         throw new Error("Server Error: Sent older slots!");
1436                 }
1437
1438                 // Create an object that can access both new slots and slots in our local chain
1439                 // without committing slots to our local chain
1440                 SlotIndexer indexer = new SlotIndexer(newSlots, buffer);
1441
1442                 // Check that the HMAC chain is not broken
1443                 checkHMACChain(indexer, newSlots);
1444
1445                 // Set to keep track of messages from clients
1446                 HashSet<Long> machineSet = new HashSet<Long>(lastMessageTable.keySet());
1447
1448                 // Process each slots data
1449                 for (Slot slot : newSlots) {
1450                         processSlot(indexer, slot, acceptUpdatesToLocal, machineSet);
1451                         updateExpectedSize();
1452                 }
1453
1454                 // If there is a gap, check to see if the server sent us everything.
1455                 if (firstSeqNum != (sequenceNumber + 1)) {
1456
1457                         // Check the size of the slots that were sent down by the server.
1458                         // Can only check the size if there was a gap
1459                         checkNumSlots(newSlots.length);
1460
1461                         // Since there was a gap every machine must have pushed a slot or must have
1462                         // a last message message.  If not then the server is hiding slots
1463                         if (!machineSet.isEmpty()) {
1464                                 throw new Error("Missing record for machines: " + machineSet);
1465                         }
1466                 }
1467
1468                 // Update the size of our local block chain.
1469                 commitNewMaxSize();
1470
1471                 // Commit new to slots to the local block chain.
1472                 for (Slot slot : newSlots) {
1473
1474                         // Insert this slot into our local block chain copy.
1475                         buffer.putSlot(slot);
1476
1477                         // Keep track of how many slots are currently live (have live data in them).
1478                         liveSlotCount++;
1479                 }
1480
1481                 // Get the sequence number of the latest slot in the system
1482                 sequenceNumber = newSlots[newSlots.length - 1].getSequenceNumber();
1483
1484                 updateLiveStateFromServer();
1485
1486                 // No Need to remember after we pulled from the server
1487                 offlineTransactionsCommittedAndAtServer.clear();
1488
1489                 // This is invalidated now
1490                 hadPartialSendToServer = false;
1491         }
1492
1493         private void updateLiveStateFromServer() {
1494                 // Process the new transaction parts
1495                 processNewTransactionParts();
1496
1497                 // Do arbitration on new transactions that were received
1498                 arbitrateFromServer();
1499
1500                 // Update all the committed keys
1501                 boolean didCommitOrSpeculate = updateCommittedTable();
1502
1503                 // Delete the transactions that are now dead
1504                 updateLiveTransactionsAndStatus();
1505
1506                 // Do speculations
1507                 didCommitOrSpeculate |= updateSpeculativeTable(didCommitOrSpeculate);
1508                 updatePendingTransactionSpeculativeTable(didCommitOrSpeculate);
1509         }
1510
1511         private void updateLiveStateFromLocal() {
1512                 // Update all the committed keys
1513                 boolean didCommitOrSpeculate = updateCommittedTable();
1514
1515                 // Delete the transactions that are now dead
1516                 updateLiveTransactionsAndStatus();
1517
1518                 // Do speculations
1519                 didCommitOrSpeculate |= updateSpeculativeTable(didCommitOrSpeculate);
1520                 updatePendingTransactionSpeculativeTable(didCommitOrSpeculate);
1521         }
1522
1523         private void initExpectedSize(long firstSequenceNumber, long numberOfSlots) {
1524
1525                 long prevslots = firstSequenceNumber;
1526                 if (didFindTableStatus) {
1527 //                      expectedsize = (prevslots < ((long) numberOfSlots)) ? (int) prevslots : expectedsize;
1528                 } else {
1529                         expectedsize = (prevslots < ((long) numberOfSlots)) ? (int) prevslots : numberOfSlots;
1530                 }
1531                 didFindTableStatus = true;
1532
1533                 currMaxSize = numberOfSlots;
1534         }
1535
1536         private void updateExpectedSize() {
1537                 expectedsize++;
1538                 if (expectedsize > currMaxSize) {
1539                         expectedsize = currMaxSize;
1540                 }
1541         }
1542
1543
1544         /**
1545          * Check the size of the block chain to make sure there are enough slots sent back by the server.
1546          * This is only called when we have a gap between the slots that we have locally and the slots
1547          * sent by the server therefore in the slots sent by the server there will be at least 1 Table
1548          * status message
1549          */
1550         private void checkNumSlots(int numberOfSlots) {
1551                 if (numberOfSlots != expectedsize) {
1552                         throw new Error("Server Error: Server did not send all slots.  Expected: " + expectedsize + " Received:" + numberOfSlots);
1553                 }
1554         }
1555
1556         private void updateCurrMaxSize(int newmaxsize) {
1557                 currMaxSize = newmaxsize;
1558         }
1559
1560
1561         /**
1562          * Update the size of of the local buffer if it is needed.
1563          */
1564         private void commitNewMaxSize() {
1565                 didFindTableStatus = false;
1566
1567                 // Resize the local slot buffer
1568                 if (numberOfSlots != currMaxSize) {
1569                         buffer.resize((int)currMaxSize);
1570                 }
1571
1572                 // Change the number of local slots to the new size
1573                 numberOfSlots = (int)currMaxSize;
1574
1575                 // Recalculate the resize threshold since the size of the local buffer has changed
1576                 setResizeThreshold();
1577         }
1578
1579         /**
1580          * Process the new transaction parts from this latest round of slots received from the server
1581          */
1582         private void processNewTransactionParts() {
1583
1584                 if (newTransactionParts.size() == 0) {
1585                         // Nothing new to process
1586                         return;
1587                 }
1588
1589                 // Iterate through all the machine Ids that we received new parts for
1590                 for (Long machineId : newTransactionParts.keySet()) {
1591                         Map<Pair<Long, Integer>, TransactionPart> parts = newTransactionParts.get(machineId);
1592
1593                         // Iterate through all the parts for that machine Id
1594                         for (Pair<Long, Integer> partId : parts.keySet()) {
1595                                 TransactionPart part = parts.get(partId);
1596
1597                                 Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(part.getArbitratorId());
1598                                 if ((lastTransactionNumber != null) && (lastTransactionNumber >= part.getSequenceNumber())) {
1599                                         // Set dead the transaction part
1600                                         part.setDead();
1601                                         continue;
1602                                 }
1603
1604                                 // Get the transaction object for that sequence number
1605                                 Transaction transaction = liveTransactionBySequenceNumberTable.get(part.getSequenceNumber());
1606
1607                                 if (transaction == null) {
1608                                         // This is a new transaction that we dont have so make a new one
1609                                         transaction = new Transaction();
1610
1611                                         // Insert this new transaction into the live tables
1612                                         liveTransactionBySequenceNumberTable.put(part.getSequenceNumber(), transaction);
1613                                         liveTransactionByTransactionIdTable.put(part.getTransactionId(), transaction);
1614                                 }
1615
1616                                 // Add that part to the transaction
1617                                 transaction.addPartDecode(part);
1618                         }
1619                 }
1620
1621                 // Clear all the new transaction parts in preparation for the next time the server sends slots
1622                 newTransactionParts.clear();
1623         }
1624
1625
1626         private long lastSeqNumArbOn = 0;
1627
1628         private void arbitrateFromServer() {
1629
1630                 if (liveTransactionBySequenceNumberTable.size() == 0) {
1631                         // Nothing to arbitrate on so move on
1632                         return;
1633                 }
1634
1635                 // Get the transaction sequence numbers and sort from oldest to newest
1636                 List<Long> transactionSequenceNumbers = new ArrayList<Long>(liveTransactionBySequenceNumberTable.keySet());
1637                 Collections.sort(transactionSequenceNumbers);
1638
1639                 // Collection of key value pairs that are
1640                 Map<IoTString, KeyValue> speculativeTableTmp = new HashMap<IoTString, KeyValue>();
1641
1642                 // The last transaction arbitrated on
1643                 long lastTransactionCommitted = -1;
1644                 Set<Abort> generatedAborts = new HashSet<Abort>();
1645
1646                 for (Long transactionSequenceNumber : transactionSequenceNumbers) {
1647                         Transaction transaction = liveTransactionBySequenceNumberTable.get(transactionSequenceNumber);
1648
1649
1650
1651                         // Check if this machine arbitrates for this transaction if not then we cant arbitrate this transaction
1652                         if (transaction.getArbitrator() != localMachineId) {
1653                                 continue;
1654                         }
1655
1656                         if (transactionSequenceNumber < lastSeqNumArbOn) {
1657                                 continue;
1658                         }
1659
1660                         if (offlineTransactionsCommittedAndAtServer.contains(transaction.getId())) {
1661                                 // We have seen this already locally so dont commit again
1662                                 continue;
1663                         }
1664
1665
1666                         if (!transaction.isComplete()) {
1667                                 // Will arbitrate in incorrect order if we continue so just break
1668                                 // Most likely this
1669                                 break;
1670                         }
1671
1672
1673                         // update the largest transaction seen by arbitrator from server
1674                         if (lastTransactionSeenFromMachineFromServer.get(transaction.getMachineId()) == null) {
1675                                 lastTransactionSeenFromMachineFromServer.put(transaction.getMachineId(), transaction.getClientLocalSequenceNumber());
1676                         } else {
1677                                 Long lastTransactionSeenFromMachine = lastTransactionSeenFromMachineFromServer.get(transaction.getMachineId());
1678                                 if (transaction.getClientLocalSequenceNumber() > lastTransactionSeenFromMachine) {
1679                                         lastTransactionSeenFromMachineFromServer.put(transaction.getMachineId(), transaction.getClientLocalSequenceNumber());
1680                                 }
1681                         }
1682
1683                         if (transaction.evaluateGuard(committedKeyValueTable, speculativeTableTmp, null)) {
1684                                 // Guard evaluated as true
1685
1686                                 // Update the local changes so we can make the commit
1687                                 for (KeyValue kv : transaction.getKeyValueUpdateSet()) {
1688                                         speculativeTableTmp.put(kv.getKey(), kv);
1689                                 }
1690
1691                                 // Update what the last transaction committed was for use in batch commit
1692                                 lastTransactionCommitted = transactionSequenceNumber;
1693                         } else {
1694                                 // Guard evaluated was false so create abort
1695
1696                                 // Create the abort
1697                                 Abort newAbort = new Abort(null,
1698                                                            transaction.getClientLocalSequenceNumber(),
1699                                                            transaction.getSequenceNumber(),
1700                                                            transaction.getMachineId(),
1701                                                            transaction.getArbitrator(),
1702                                                            localArbitrationSequenceNumber);
1703                                 localArbitrationSequenceNumber++;
1704
1705                                 generatedAborts.add(newAbort);
1706
1707                                 // Insert the abort so we can process
1708                                 processEntry(newAbort);
1709                         }
1710
1711                         lastSeqNumArbOn = transactionSequenceNumber;
1712
1713                         // liveTransactionBySequenceNumberTable.remove(transactionSequenceNumber);
1714                 }
1715
1716                 Commit newCommit = null;
1717
1718                 // If there is something to commit
1719                 if (speculativeTableTmp.size() != 0) {
1720
1721                         // Create the commit and increment the commit sequence number
1722                         newCommit = new Commit(localArbitrationSequenceNumber, localMachineId, lastTransactionCommitted);
1723                         localArbitrationSequenceNumber++;
1724
1725                         // Add all the new keys to the commit
1726                         for (KeyValue kv : speculativeTableTmp.values()) {
1727                                 newCommit.addKV(kv);
1728                         }
1729
1730                         // create the commit parts
1731                         newCommit.createCommitParts();
1732
1733                         // Append all the commit parts to the end of the pending queue waiting for sending to the server
1734
1735                         // Insert the commit so we can process it
1736                         for (CommitPart commitPart : newCommit.getParts().values()) {
1737                                 processEntry(commitPart);
1738                         }
1739                 }
1740
1741                 if ((newCommit != null) || (generatedAborts.size() > 0)) {
1742                         ArbitrationRound arbitrationRound = new ArbitrationRound(newCommit, generatedAborts);
1743                         pendingSendArbitrationRounds.add(arbitrationRound);
1744
1745                         if (compactArbitrationData()) {
1746                                 ArbitrationRound newArbitrationRound = pendingSendArbitrationRounds.get(pendingSendArbitrationRounds.size() - 1);
1747                                 if (newArbitrationRound.getCommit() != null) {
1748                                         for (CommitPart commitPart : newArbitrationRound.getCommit().getParts().values()) {
1749                                                 processEntry(commitPart);
1750                                         }
1751                                 }
1752                         }
1753                 }
1754         }
1755
1756         private Pair<Boolean, Boolean> arbitrateOnLocalTransaction(Transaction transaction) {
1757
1758                 // Check if this machine arbitrates for this transaction if not then we cant arbitrate this transaction
1759                 if (transaction.getArbitrator() != localMachineId) {
1760                         return new Pair<Boolean, Boolean>(false, false);
1761                 }
1762
1763                 if (!transaction.isComplete()) {
1764                         // Will arbitrate in incorrect order if we continue so just break
1765                         // Most likely this
1766                         return new Pair<Boolean, Boolean>(false, false);
1767                 }
1768
1769                 if (transaction.getMachineId() != localMachineId) {
1770                         // dont do this check for local transactions
1771                         if (lastTransactionSeenFromMachineFromServer.get(transaction.getMachineId()) != null) {
1772                                 if (lastTransactionSeenFromMachineFromServer.get(transaction.getMachineId()) > transaction.getClientLocalSequenceNumber()) {
1773                                         // We've have already seen this from the server
1774                                         return new Pair<Boolean, Boolean>(false, false);
1775                                 }
1776                         }
1777                 }
1778
1779                 if (transaction.evaluateGuard(committedKeyValueTable, null, null)) {
1780                         // Guard evaluated as true
1781
1782                         // Create the commit and increment the commit sequence number
1783                         Commit newCommit = new Commit(localArbitrationSequenceNumber, localMachineId, -1);
1784                         localArbitrationSequenceNumber++;
1785
1786                         // Update the local changes so we can make the commit
1787                         for (KeyValue kv : transaction.getKeyValueUpdateSet()) {
1788                                 newCommit.addKV(kv);
1789                         }
1790
1791                         // create the commit parts
1792                         newCommit.createCommitParts();
1793
1794                         // Append all the commit parts to the end of the pending queue waiting for sending to the server
1795                         ArbitrationRound arbitrationRound = new ArbitrationRound(newCommit, new HashSet<Abort>());
1796                         pendingSendArbitrationRounds.add(arbitrationRound);
1797
1798                         if (compactArbitrationData()) {
1799                                 ArbitrationRound newArbitrationRound = pendingSendArbitrationRounds.get(pendingSendArbitrationRounds.size() - 1);
1800                                 for (CommitPart commitPart : newArbitrationRound.getCommit().getParts().values()) {
1801                                         processEntry(commitPart);
1802                                 }
1803                         } else {
1804                                 // Insert the commit so we can process it
1805                                 for (CommitPart commitPart : newCommit.getParts().values()) {
1806                                         processEntry(commitPart);
1807                                 }
1808                         }
1809
1810                         if (transaction.getMachineId() == localMachineId) {
1811                                 TransactionStatus status = transaction.getTransactionStatus();
1812                                 if (status != null) {
1813                                         status.setStatus(TransactionStatus.StatusCommitted);
1814                                 }
1815                         }
1816
1817                         updateLiveStateFromLocal();
1818                         return new Pair<Boolean, Boolean>(true, true);
1819                 } else {
1820
1821                         if (transaction.getMachineId() == localMachineId) {
1822                                 // For locally created messages update the status
1823
1824                                 // Guard evaluated was false so create abort
1825                                 TransactionStatus status = transaction.getTransactionStatus();
1826                                 if (status != null) {
1827                                         status.setStatus(TransactionStatus.StatusAborted);
1828                                 }
1829                         } else {
1830                                 Set addAbortSet = new HashSet<Abort>();
1831
1832
1833                                 // Create the abort
1834                                 Abort newAbort = new Abort(null,
1835                                                            transaction.getClientLocalSequenceNumber(),
1836                                                            -1,
1837                                                            transaction.getMachineId(),
1838                                                            transaction.getArbitrator(),
1839                                                            localArbitrationSequenceNumber);
1840                                 localArbitrationSequenceNumber++;
1841
1842                                 addAbortSet.add(newAbort);
1843
1844
1845                                 // Append all the commit parts to the end of the pending queue waiting for sending to the server
1846                                 ArbitrationRound arbitrationRound = new ArbitrationRound(null, addAbortSet);
1847                                 pendingSendArbitrationRounds.add(arbitrationRound);
1848
1849                                 if (compactArbitrationData()) {
1850                                         ArbitrationRound newArbitrationRound = pendingSendArbitrationRounds.get(pendingSendArbitrationRounds.size() - 1);
1851                                         for (CommitPart commitPart : newArbitrationRound.getCommit().getParts().values()) {
1852                                                 processEntry(commitPart);
1853                                         }
1854                                 }
1855                         }
1856
1857                         updateLiveStateFromLocal();
1858                         return new Pair<Boolean, Boolean>(true, false);
1859                 }
1860         }
1861
1862         /**
1863          * Compacts the arbitration data my merging commits and aggregating aborts so that a single large push of commits can be done instead of many small updates
1864          */
1865         private boolean compactArbitrationData() {
1866
1867                 if (pendingSendArbitrationRounds.size() < 2) {
1868                         // Nothing to compact so do nothing
1869                         return false;
1870                 }
1871
1872                 ArbitrationRound lastRound = pendingSendArbitrationRounds.get(pendingSendArbitrationRounds.size() - 1);
1873                 if (lastRound.didSendPart()) {
1874                         return false;
1875                 }
1876
1877                 boolean hadCommit = (lastRound.getCommit() == null);
1878                 boolean gotNewCommit = false;
1879
1880                 int numberToDelete = 1;
1881                 while (numberToDelete < pendingSendArbitrationRounds.size()) {
1882                         ArbitrationRound round = pendingSendArbitrationRounds.get(pendingSendArbitrationRounds.size() - numberToDelete - 1);
1883
1884                         if (round.isFull() || round.didSendPart()) {
1885                                 // Stop since there is a part that cannot be compacted and we need to compact in order
1886                                 break;
1887                         }
1888
1889                         if (round.getCommit() == null) {
1890
1891                                 // Try compacting aborts only
1892                                 int newSize = round.getCurrentSize() + lastRound.getAbortsCount();
1893                                 if (newSize > ArbitrationRound.MAX_PARTS) {
1894                                         // Cant compact since it would be too large
1895                                         break;
1896                                 }
1897                                 lastRound.addAborts(round.getAborts());
1898                         } else {
1899
1900                                 // Create a new larger commit
1901                                 Commit newCommit = Commit.merge(lastRound.getCommit(), round.getCommit(), localArbitrationSequenceNumber);
1902                                 localArbitrationSequenceNumber++;
1903
1904                                 // Create the commit parts so that we can count them
1905                                 newCommit.createCommitParts();
1906
1907                                 // Calculate the new size of the parts
1908                                 int newSize = newCommit.getNumberOfParts();
1909                                 newSize += lastRound.getAbortsCount();
1910                                 newSize += round.getAbortsCount();
1911
1912                                 if (newSize > ArbitrationRound.MAX_PARTS) {
1913                                         // Cant compact since it would be too large
1914                                         break;
1915                                 }
1916
1917                                 // Set the new compacted part
1918                                 lastRound.setCommit(newCommit);
1919                                 lastRound.addAborts(round.getAborts());
1920                                 gotNewCommit = true;
1921                         }
1922
1923                         numberToDelete++;
1924                 }
1925
1926                 if (numberToDelete != 1) {
1927                         // If there is a compaction
1928
1929                         // Delete the previous pieces that are now in the new compacted piece
1930                         if (numberToDelete == pendingSendArbitrationRounds.size()) {
1931                                 pendingSendArbitrationRounds.clear();
1932                         } else {
1933                                 for (int i = 0; i < numberToDelete; i++) {
1934                                         pendingSendArbitrationRounds.remove(pendingSendArbitrationRounds.size() - 1);
1935                                 }
1936                         }
1937
1938                         // Add the new compacted into the pending to send list
1939                         pendingSendArbitrationRounds.add(lastRound);
1940
1941                         // Should reinsert into the commit processor
1942                         if (hadCommit && gotNewCommit) {
1943                                 return true;
1944                         }
1945                 }
1946
1947                 return false;
1948         }
1949         // private boolean compactArbitrationData() {
1950         //      return false;
1951         // }
1952
1953         /**
1954          * Update all the commits and the committed tables, sets dead the dead transactions
1955          */
1956         private boolean updateCommittedTable() {
1957
1958                 if (newCommitParts.size() == 0) {
1959                         // Nothing new to process
1960                         return false;
1961                 }
1962
1963                 // Iterate through all the machine Ids that we received new parts for
1964                 for (Long machineId : newCommitParts.keySet()) {
1965                         Map<Pair<Long, Integer>, CommitPart> parts = newCommitParts.get(machineId);
1966
1967                         // Iterate through all the parts for that machine Id
1968                         for (Pair<Long, Integer> partId : parts.keySet()) {
1969                                 CommitPart part = parts.get(partId);
1970
1971                                 // Get the transaction object for that sequence number
1972                                 Map<Long, Commit> commitForClientTable = liveCommitsTable.get(part.getMachineId());
1973
1974                                 if (commitForClientTable == null) {
1975                                         // This is the first commit from this device
1976                                         commitForClientTable = new HashMap<Long, Commit>();
1977                                         liveCommitsTable.put(part.getMachineId(), commitForClientTable);
1978                                 }
1979
1980                                 Commit commit = commitForClientTable.get(part.getSequenceNumber());
1981
1982                                 if (commit == null) {
1983                                         // This is a new commit that we dont have so make a new one
1984                                         commit = new Commit();
1985
1986                                         // Insert this new commit into the live tables
1987                                         commitForClientTable.put(part.getSequenceNumber(), commit);
1988                                 }
1989
1990                                 // Add that part to the commit
1991                                 commit.addPartDecode(part);
1992                         }
1993                 }
1994
1995                 // Clear all the new commits parts in preparation for the next time the server sends slots
1996                 newCommitParts.clear();
1997
1998                 // If we process a new commit keep track of it for future use
1999                 boolean didProcessANewCommit = false;
2000
2001                 // Process the commits one by one
2002                 for (Long arbitratorId : liveCommitsTable.keySet()) {
2003
2004                         // Get all the commits for a specific arbitrator
2005                         Map<Long, Commit> commitForClientTable = liveCommitsTable.get(arbitratorId);
2006
2007                         // Sort the commits in order
2008                         List<Long> commitSequenceNumbers = new ArrayList<Long>(commitForClientTable.keySet());
2009                         Collections.sort(commitSequenceNumbers);
2010
2011                         // Get the last commit seen from this arbitrator
2012                         long lastCommitSeenSequenceNumber = -1;
2013                         if (lastCommitSeenSequenceNumberByArbitratorTable.get(arbitratorId) != null) {
2014                                 lastCommitSeenSequenceNumber = lastCommitSeenSequenceNumberByArbitratorTable.get(arbitratorId);
2015                         }
2016
2017                         // Go through each new commit one by one
2018                         for (int i = 0; i < commitSequenceNumbers.size(); i++) {
2019                                 Long commitSequenceNumber = commitSequenceNumbers.get(i);
2020                                 Commit commit = commitForClientTable.get(commitSequenceNumber);
2021
2022                                 // Special processing if a commit is not complete
2023                                 if (!commit.isComplete()) {
2024                                         if (i == (commitSequenceNumbers.size() - 1)) {
2025                                                 // If there is an incomplete commit and this commit is the latest one seen then this commit cannot be processed and there are no other commits
2026                                                 break;
2027                                         } else {
2028                                                 // This is a commit that was already dead but parts of it are still in the block chain (not flushed out yet).
2029                                                 // Delete it and move on
2030                                                 commit.setDead();
2031                                                 commitForClientTable.remove(commit.getSequenceNumber());
2032                                                 continue;
2033                                         }
2034                                 }
2035
2036                                 // Update the last transaction that was updated if we can
2037                                 if (commit.getTransactionSequenceNumber() != -1) {
2038                                         Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(commit.getMachineId());
2039
2040                                         // Update the last transaction sequence number that the arbitrator arbitrated on
2041                                         if ((lastTransactionNumber == null) || (lastTransactionNumber < commit.getTransactionSequenceNumber())) {
2042                                                 lastArbitratedTransactionNumberByArbitratorTable.put(commit.getMachineId(), commit.getTransactionSequenceNumber());
2043                                         }
2044                                 }
2045
2046                                 // Update the last arbitration data that we have seen so far
2047                                 if (lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(commit.getMachineId()) != null) {
2048
2049                                         long lastArbitrationSequenceNumber = lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(commit.getMachineId());
2050                                         if (commit.getSequenceNumber() > lastArbitrationSequenceNumber) {
2051                                                 // Is larger
2052                                                 lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.put(commit.getMachineId(), commit.getSequenceNumber());
2053                                         }
2054                                 } else {
2055                                         // Never seen any data from this arbitrator so record the first one
2056                                         lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.put(commit.getMachineId(), commit.getSequenceNumber());
2057                                 }
2058
2059                                 // We have already seen this commit before so need to do the full processing on this commit
2060                                 if (commit.getSequenceNumber() <= lastCommitSeenSequenceNumber) {
2061
2062                                         // Update the last transaction that was updated if we can
2063                                         if (commit.getTransactionSequenceNumber() != -1) {
2064                                                 Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(commit.getMachineId());
2065
2066                                                 // Update the last transaction sequence number that the arbitrator arbitrated on
2067                                                 if ((lastTransactionNumber == null) || (lastTransactionNumber < commit.getTransactionSequenceNumber())) {
2068                                                         lastArbitratedTransactionNumberByArbitratorTable.put(commit.getMachineId(), commit.getTransactionSequenceNumber());
2069                                                 }
2070                                         }
2071
2072                                         continue;
2073                                 }
2074
2075                                 // If we got here then this is a brand new commit and needs full processing
2076
2077                                 // Get what commits should be edited, these are the commits that have live values for their keys
2078                                 Set<Commit> commitsToEdit = new HashSet<Commit>();
2079                                 for (KeyValue kv : commit.getKeyValueUpdateSet()) {
2080                                         commitsToEdit.add(liveCommitsByKeyTable.get(kv.getKey()));
2081                                 }
2082                                 commitsToEdit.remove(null); // remove null since it could be in this set
2083
2084                                 // Update each previous commit that needs to be updated
2085                                 for (Commit previousCommit : commitsToEdit) {
2086
2087                                         // Only bother with live commits (TODO: Maybe remove this check)
2088                                         if (previousCommit.isLive()) {
2089
2090                                                 // Update which keys in the old commits are still live
2091                                                 for (KeyValue kv : commit.getKeyValueUpdateSet()) {
2092                                                         previousCommit.invalidateKey(kv.getKey());
2093                                                 }
2094
2095                                                 // if the commit is now dead then remove it
2096                                                 if (!previousCommit.isLive()) {
2097                                                         commitForClientTable.remove(previousCommit);
2098                                                 }
2099                                         }
2100                                 }
2101
2102                                 // Update the last seen sequence number from this arbitrator
2103                                 if (lastCommitSeenSequenceNumberByArbitratorTable.get(commit.getMachineId()) != null) {
2104                                         if (commit.getSequenceNumber() > lastCommitSeenSequenceNumberByArbitratorTable.get(commit.getMachineId())) {
2105                                                 lastCommitSeenSequenceNumberByArbitratorTable.put(commit.getMachineId(), commit.getSequenceNumber());
2106                                         }
2107                                 } else {
2108                                         lastCommitSeenSequenceNumberByArbitratorTable.put(commit.getMachineId(), commit.getSequenceNumber());
2109                                 }
2110
2111                                 // We processed a new commit that we havent seen before
2112                                 didProcessANewCommit = true;
2113
2114                                 // Update the committed table of keys and which commit is using which key
2115                                 for (KeyValue kv : commit.getKeyValueUpdateSet()) {
2116                                         committedKeyValueTable.put(kv.getKey(), kv);
2117                                         liveCommitsByKeyTable.put(kv.getKey(), commit);
2118                                 }
2119                         }
2120                 }
2121
2122                 return didProcessANewCommit;
2123         }
2124
2125         /**
2126          * Create the speculative table from transactions that are still live and have come from the cloud
2127          */
2128         private boolean updateSpeculativeTable(boolean didProcessNewCommits) {
2129                 if (liveTransactionBySequenceNumberTable.keySet().size() == 0) {
2130                         // There is nothing to speculate on
2131                         return false;
2132                 }
2133
2134                 // Create a list of the transaction sequence numbers and sort them from oldest to newest
2135                 List<Long> transactionSequenceNumbersSorted = new ArrayList<Long>(liveTransactionBySequenceNumberTable.keySet());
2136                 Collections.sort(transactionSequenceNumbersSorted);
2137
2138                 boolean hasGapInTransactionSequenceNumbers = transactionSequenceNumbersSorted.get(0) != oldestTransactionSequenceNumberSpeculatedOn;
2139
2140
2141                 if (hasGapInTransactionSequenceNumbers || didProcessNewCommits) {
2142                         // If there is a gap in the transaction sequence numbers then there was a commit or an abort of a transaction
2143                         // OR there was a new commit (Could be from offline commit) so a redo the speculation from scratch
2144
2145                         // Start from scratch
2146                         speculatedKeyValueTable.clear();
2147                         lastTransactionSequenceNumberSpeculatedOn = -1;
2148                         oldestTransactionSequenceNumberSpeculatedOn = -1;
2149
2150                 }
2151
2152                 // Remember the front of the transaction list
2153                 oldestTransactionSequenceNumberSpeculatedOn = transactionSequenceNumbersSorted.get(0);
2154
2155                 // Find where to start arbitration from
2156                 int startIndex = transactionSequenceNumbersSorted.indexOf(lastTransactionSequenceNumberSpeculatedOn) + 1;
2157
2158                 if (startIndex >= transactionSequenceNumbersSorted.size()) {
2159                         // Make sure we are not out of bounds
2160                         return false; // did not speculate
2161                 }
2162
2163                 Set<Long> incompleteTransactionArbitrator = new HashSet<Long>();
2164                 boolean didSkip = true;
2165
2166                 for (int i = startIndex; i < transactionSequenceNumbersSorted.size(); i++) {
2167                         long transactionSequenceNumber = transactionSequenceNumbersSorted.get(i);
2168                         Transaction transaction = liveTransactionBySequenceNumberTable.get(transactionSequenceNumber);
2169
2170                         if (!transaction.isComplete()) {
2171                                 // If there is an incomplete transaction then there is nothing we can do
2172                                 // add this transactions arbitrator to the list of arbitrators we should ignore
2173                                 incompleteTransactionArbitrator.add(transaction.getArbitrator());
2174                                 didSkip = true;
2175                                 continue;
2176                         }
2177
2178                         if (incompleteTransactionArbitrator.contains(transaction.getArbitrator())) {
2179                                 continue;
2180                         }
2181
2182                         lastTransactionSequenceNumberSpeculatedOn = transactionSequenceNumber;
2183
2184                         if (transaction.evaluateGuard(committedKeyValueTable, speculatedKeyValueTable, null)) {
2185                                 // Guard evaluated to true so update the speculative table
2186                                 for (KeyValue kv : transaction.getKeyValueUpdateSet()) {
2187                                         speculatedKeyValueTable.put(kv.getKey(), kv);
2188                                 }
2189                         }
2190                 }
2191
2192                 if (didSkip) {
2193                         // Since there was a skip we need to redo the speculation next time around
2194                         lastTransactionSequenceNumberSpeculatedOn = -1;
2195                         oldestTransactionSequenceNumberSpeculatedOn = -1;
2196                 }
2197
2198                 // We did some speculation
2199                 return true;
2200         }
2201
2202         /**
2203          * Create the pending transaction speculative table from transactions that are still in the pending transaction buffer
2204          */
2205         private void updatePendingTransactionSpeculativeTable(boolean didProcessNewCommitsOrSpeculate) {
2206                 if (pendingTransactionQueue.size() == 0) {
2207                         // There is nothing to speculate on
2208                         return;
2209                 }
2210
2211
2212                 if (didProcessNewCommitsOrSpeculate || (firstPendingTransaction != pendingTransactionQueue.get(0))) {
2213                         // need to reset on the pending speculation
2214                         lastPendingTransactionSpeculatedOn = null;
2215                         firstPendingTransaction = pendingTransactionQueue.get(0);
2216                         pendingTransactionSpeculatedKeyValueTable.clear();
2217                 }
2218
2219                 // Find where to start arbitration from
2220                 int startIndex = pendingTransactionQueue.indexOf(firstPendingTransaction) + 1;
2221
2222                 if (startIndex >= pendingTransactionQueue.size()) {
2223                         // Make sure we are not out of bounds
2224                         return;
2225                 }
2226
2227                 for (int i = startIndex; i < pendingTransactionQueue.size(); i++) {
2228                         Transaction transaction = pendingTransactionQueue.get(i);
2229
2230                         lastPendingTransactionSpeculatedOn = transaction;
2231
2232                         if (transaction.evaluateGuard(committedKeyValueTable, speculatedKeyValueTable, pendingTransactionSpeculatedKeyValueTable)) {
2233                                 // Guard evaluated to true so update the speculative table
2234                                 for (KeyValue kv : transaction.getKeyValueUpdateSet()) {
2235                                         pendingTransactionSpeculatedKeyValueTable.put(kv.getKey(), kv);
2236                                 }
2237                         }
2238                 }
2239         }
2240
2241         /**
2242          * Set dead and remove from the live transaction tables the transactions that are dead
2243          */
2244         private void updateLiveTransactionsAndStatus() {
2245
2246                 // Go through each of the transactions
2247                 for (Iterator<Map.Entry<Long, Transaction>> iter = liveTransactionBySequenceNumberTable.entrySet().iterator(); iter.hasNext();) {
2248                         Transaction transaction = iter.next().getValue();
2249
2250                         // Check if the transaction is dead
2251                         Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(transaction.getArbitrator());
2252                         if ((lastTransactionNumber != null) && (lastTransactionNumber >= transaction.getSequenceNumber())) {
2253
2254                                 // Set dead the transaction
2255                                 transaction.setDead();
2256
2257                                 // Remove the transaction from the live table
2258                                 iter.remove();
2259                                 liveTransactionByTransactionIdTable.remove(transaction.getId());
2260                         }
2261                 }
2262
2263                 // Go through each of the transactions
2264                 for (Iterator<Map.Entry<Long, TransactionStatus>> iter = outstandingTransactionStatus.entrySet().iterator(); iter.hasNext();) {
2265                         TransactionStatus status = iter.next().getValue();
2266
2267                         // Check if the transaction is dead
2268                         Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(status.getTransactionArbitrator());
2269                         if ((lastTransactionNumber != null) && (lastTransactionNumber >= status.getTransactionSequenceNumber())) {
2270
2271                                 // Set committed
2272                                 status.setStatus(TransactionStatus.StatusCommitted);
2273
2274                                 // Remove
2275                                 iter.remove();
2276                         }
2277                 }
2278         }
2279
2280         /**
2281          * Process this slot, entry by entry.  Also update the latest message sent by slot
2282          */
2283         private void processSlot(SlotIndexer indexer, Slot slot, boolean acceptUpdatesToLocal, HashSet<Long> machineSet) {
2284
2285                 // Update the last message seen
2286                 updateLastMessage(slot.getMachineID(), slot.getSequenceNumber(), slot, acceptUpdatesToLocal, machineSet);
2287
2288                 // Process each entry in the slot
2289                 for (Entry entry : slot.getEntries()) {
2290                         switch (entry.getType()) {
2291
2292                         case Entry.TypeCommitPart:
2293                                 processEntry((CommitPart)entry);
2294                                 break;
2295
2296                         case Entry.TypeAbort:
2297                                 processEntry((Abort)entry);
2298                                 break;
2299
2300                         case Entry.TypeTransactionPart:
2301                                 processEntry((TransactionPart)entry);
2302                                 break;
2303
2304                         case Entry.TypeNewKey:
2305                                 processEntry((NewKey)entry);
2306                                 break;
2307
2308                         case Entry.TypeLastMessage:
2309                                 processEntry((LastMessage)entry, machineSet);
2310                                 break;
2311
2312                         case Entry.TypeRejectedMessage:
2313                                 processEntry((RejectedMessage)entry, indexer);
2314                                 break;
2315
2316                         case Entry.TypeTableStatus:
2317                                 processEntry((TableStatus)entry, slot.getSequenceNumber());
2318                                 break;
2319
2320                         default:
2321                                 throw new Error("Unrecognized type: " + entry.getType());
2322                         }
2323                 }
2324         }
2325
2326         /**
2327          * Update the last message that was sent for a machine Id
2328          */
2329         private void processEntry(LastMessage entry, HashSet<Long> machineSet) {
2330                 // Update what the last message received by a machine was
2331                 updateLastMessage(entry.getMachineID(), entry.getSequenceNumber(), entry, false, machineSet);
2332         }
2333
2334         /**
2335          * Add the new key to the arbitrators table and update the set of live new keys (in case of a rescued new key message)
2336          */
2337         private void processEntry(NewKey entry) {
2338
2339                 // Update the arbitrator table with the new key information
2340                 arbitratorTable.put(entry.getKey(), entry.getMachineID());
2341
2342                 // Update what the latest live new key is
2343                 NewKey oldNewKey = liveNewKeyTable.put(entry.getKey(), entry);
2344                 if (oldNewKey != null) {
2345                         // Delete the old new key messages
2346                         oldNewKey.setDead();
2347                 }
2348         }
2349
2350         /**
2351          * Process new table status entries and set dead the old ones as new ones come in.
2352          * keeps track of the largest and smallest table status seen in this current round
2353          * of updating the local copy of the block chain
2354          */
2355         private void processEntry(TableStatus entry, long seq) {
2356                 int newNumSlots = entry.getMaxSlots();
2357                 updateCurrMaxSize(newNumSlots);
2358
2359                 initExpectedSize(seq, newNumSlots);
2360
2361                 if (liveTableStatus != null) {
2362                         // We have a larger table status so the old table status is no longer alive
2363                         liveTableStatus.setDead();
2364                 }
2365
2366                 // Make this new table status the latest alive table status
2367                 liveTableStatus = entry;
2368         }
2369
2370         /**
2371          * Check old messages to see if there is a block chain violation. Also
2372          */
2373         private void processEntry(RejectedMessage entry, SlotIndexer indexer) {
2374                 long oldSeqNum = entry.getOldSeqNum();
2375                 long newSeqNum = entry.getNewSeqNum();
2376                 boolean isequal = entry.getEqual();
2377                 long machineId = entry.getMachineID();
2378                 long seq = entry.getSequenceNumber();
2379
2380
2381                 // Check if we have messages that were supposed to be rejected in our local block chain
2382                 for (long seqNum = oldSeqNum; seqNum <= newSeqNum; seqNum++) {
2383
2384                         // Get the slot
2385                         Slot slot = indexer.getSlot(seqNum);
2386
2387                         if (slot != null) {
2388                                 // If we have this slot make sure that it was not supposed to be a rejected slot
2389
2390                                 long slotMachineId = slot.getMachineID();
2391                                 if (isequal != (slotMachineId == machineId)) {
2392                                         throw new Error("Server Error: Trying to insert rejected message for slot " + seqNum);
2393                                 }
2394                         }
2395                 }
2396
2397
2398                 // Create a list of clients to watch until they see this rejected message entry.
2399                 HashSet<Long> deviceWatchSet = new HashSet<Long>();
2400                 for (Map.Entry<Long, Pair<Long, Liveness>> lastMessageEntry : lastMessageTable.entrySet()) {
2401
2402                         // Machine ID for the last message entry
2403                         long lastMessageEntryMachineId = lastMessageEntry.getKey();
2404
2405                         // We've seen it, don't need to continue to watch.  Our next
2406                         // message will implicitly acknowledge it.
2407                         if (lastMessageEntryMachineId == localMachineId) {
2408                                 continue;
2409                         }
2410
2411                         Pair<Long, Liveness> lastMessageValue = lastMessageEntry.getValue();
2412                         long entrySequenceNumber = lastMessageValue.getFirst();
2413
2414                         if (entrySequenceNumber < seq) {
2415
2416                                 // Add this rejected message to the set of messages that this machine ID did not see yet
2417                                 addWatchList(lastMessageEntryMachineId, entry);
2418
2419                                 // This client did not see this rejected message yet so add it to the watch set to monitor
2420                                 deviceWatchSet.add(lastMessageEntryMachineId);
2421                         }
2422                 }
2423
2424                 if (deviceWatchSet.isEmpty()) {
2425                         // This rejected message has been seen by all the clients so
2426                         entry.setDead();
2427                 } else {
2428                         // We need to watch this rejected message
2429                         entry.setWatchSet(deviceWatchSet);
2430                 }
2431         }
2432
2433         /**
2434          * Check if this abort is live, if not then save it so we can kill it later.
2435          * update the last transaction number that was arbitrated on.
2436          */
2437         private void processEntry(Abort entry) {
2438
2439
2440                 if (entry.getTransactionSequenceNumber() != -1) {
2441                         // update the transaction status if it was sent to the server
2442                         TransactionStatus status = outstandingTransactionStatus.remove(entry.getTransactionSequenceNumber());
2443                         if (status != null) {
2444                                 status.setStatus(TransactionStatus.StatusAborted);
2445                         }
2446                 }
2447
2448                 // Abort has not been seen by the client it is for yet so we need to keep track of it
2449                 Abort previouslySeenAbort = liveAbortTable.put(entry.getAbortId(), entry);
2450                 if (previouslySeenAbort != null) {
2451                         previouslySeenAbort.setDead(); // Delete old version of the abort since we got a rescued newer version
2452                 }
2453
2454                 if (entry.getTransactionArbitrator() == localMachineId) {
2455                         liveAbortsGeneratedByLocal.put(entry.getArbitratorLocalSequenceNumber(), entry);
2456                 }
2457
2458                 if ((entry.getSequenceNumber() != -1) && (lastMessageTable.get(entry.getTransactionMachineId()).getFirst() >= entry.getSequenceNumber())) {
2459
2460                         // The machine already saw this so it is dead
2461                         entry.setDead();
2462                         liveAbortTable.remove(entry.getAbortId());
2463
2464                         if (entry.getTransactionArbitrator() == localMachineId) {
2465                                 liveAbortsGeneratedByLocal.remove(entry.getArbitratorLocalSequenceNumber());
2466                         }
2467
2468                         return;
2469                 }
2470
2471
2472
2473
2474                 // Update the last arbitration data that we have seen so far
2475                 if (lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(entry.getTransactionArbitrator()) != null) {
2476
2477                         long lastArbitrationSequenceNumber = lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.get(entry.getTransactionArbitrator());
2478                         if (entry.getSequenceNumber() > lastArbitrationSequenceNumber) {
2479                                 // Is larger
2480                                 lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.put(entry.getTransactionArbitrator(), entry.getSequenceNumber());
2481                         }
2482                 } else {
2483                         // Never seen any data from this arbitrator so record the first one
2484                         lastArbitrationDataLocalSequenceNumberSeenFromArbitrator.put(entry.getTransactionArbitrator(), entry.getSequenceNumber());
2485                 }
2486
2487
2488                 // Set dead a transaction if we can
2489                 Transaction transactionToSetDead = liveTransactionByTransactionIdTable.remove(new Pair<Long, Long>(entry.getTransactionMachineId(), entry.getTransactionClientLocalSequenceNumber()));
2490                 if (transactionToSetDead != null) {
2491                         liveTransactionBySequenceNumberTable.remove(transactionToSetDead.getSequenceNumber());
2492                 }
2493
2494                 // Update the last transaction sequence number that the arbitrator arbitrated on
2495                 Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(entry.getTransactionArbitrator());
2496                 if ((lastTransactionNumber == null) || (lastTransactionNumber < entry.getTransactionSequenceNumber())) {
2497
2498                         // Is a valid one
2499                         if (entry.getTransactionSequenceNumber() != -1) {
2500                                 lastArbitratedTransactionNumberByArbitratorTable.put(entry.getTransactionArbitrator(), entry.getTransactionSequenceNumber());
2501                         }
2502                 }
2503         }
2504
2505         /**
2506          * Set dead the transaction part if that transaction is dead and keep track of all new parts
2507          */
2508         private void processEntry(TransactionPart entry) {
2509                 // Check if we have already seen this transaction and set it dead OR if it is not alive
2510                 Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(entry.getArbitratorId());
2511                 if ((lastTransactionNumber != null) && (lastTransactionNumber >= entry.getSequenceNumber())) {
2512                         // This transaction is dead, it was already committed or aborted
2513                         entry.setDead();
2514                         return;
2515                 }
2516
2517                 // This part is still alive
2518                 Map<Pair<Long, Integer>, TransactionPart> transactionPart = newTransactionParts.get(entry.getMachineId());
2519
2520                 if (transactionPart == null) {
2521                         // Dont have a table for this machine Id yet so make one
2522                         transactionPart = new HashMap<Pair<Long, Integer>, TransactionPart>();
2523                         newTransactionParts.put(entry.getMachineId(), transactionPart);
2524                 }
2525
2526                 // Update the part and set dead ones we have already seen (got a rescued version)
2527                 TransactionPart previouslySeenPart = transactionPart.put(entry.getPartId(), entry);
2528                 if (previouslySeenPart != null) {
2529                         previouslySeenPart.setDead();
2530                 }
2531         }
2532
2533         /**
2534          * Process new commit entries and save them for future use.  Delete duplicates
2535          */
2536         private void processEntry(CommitPart entry) {
2537
2538
2539                 // Update the last transaction that was updated if we can
2540                 if (entry.getTransactionSequenceNumber() != -1) {
2541                         Long lastTransactionNumber = lastArbitratedTransactionNumberByArbitratorTable.get(entry.getMachineId());
2542
2543                         // Update the last transaction sequence number that the arbitrator arbitrated on
2544                         if ((lastTransactionNumber == null) || (lastTransactionNumber < entry.getTransactionSequenceNumber())) {
2545                                 lastArbitratedTransactionNumberByArbitratorTable.put(entry.getMachineId(), entry.getTransactionSequenceNumber());
2546                         }
2547                 }
2548
2549
2550
2551
2552                 Map<Pair<Long, Integer>, CommitPart> commitPart = newCommitParts.get(entry.getMachineId());
2553
2554                 if (commitPart == null) {
2555                         // Don't have a table for this machine Id yet so make one
2556                         commitPart = new HashMap<Pair<Long, Integer>, CommitPart>();
2557                         newCommitParts.put(entry.getMachineId(), commitPart);
2558                 }
2559
2560                 // Update the part and set dead ones we have already seen (got a rescued version)
2561                 CommitPart previouslySeenPart = commitPart.put(entry.getPartId(), entry);
2562                 if (previouslySeenPart != null) {
2563                         previouslySeenPart.setDead();
2564                 }
2565         }
2566
2567         /**
2568          * Update the last message seen table.  Update and set dead the appropriate RejectedMessages as clients see them.
2569          * Updates the live aborts, removes those that are dead and sets them dead.
2570          * Check that the last message seen is correct and that there is no mismatch of our own last message or that
2571          * other clients have not had a rollback on the last message.
2572          */
2573         private void updateLastMessage(long machineId, long seqNum, Liveness liveness, boolean acceptUpdatesToLocal, HashSet<Long> machineSet) {
2574
2575                 // We have seen this machine ID
2576                 machineSet.remove(machineId);
2577
2578                 // Get the set of rejected messages that this machine Id is has not seen yet
2579                 HashSet<RejectedMessage> watchset = rejectedMessageWatchListTable.get(machineId);
2580
2581                 // If there is a rejected message that this machine Id has not seen yet
2582                 if (watchset != null) {
2583
2584                         // Go through each rejected message that this machine Id has not seen yet
2585                         for (Iterator<RejectedMessage> rmit = watchset.iterator(); rmit.hasNext(); ) {
2586
2587                                 RejectedMessage rm = rmit.next();
2588
2589                                 // If this machine Id has seen this rejected message...
2590                                 if (rm.getSequenceNumber() <= seqNum) {
2591
2592                                         // Remove it from our watchlist
2593                                         rmit.remove();
2594
2595                                         // Decrement machines that need to see this notification
2596                                         rm.removeWatcher(machineId);
2597                                 }
2598                         }
2599                 }
2600
2601                 // Set dead the abort
2602                 for (Iterator<Map.Entry<Pair<Long, Long>, Abort>> i = liveAbortTable.entrySet().iterator(); i.hasNext();) {
2603                         Abort abort = i.next().getValue();
2604
2605                         if ((abort.getTransactionMachineId() == machineId) && (abort.getSequenceNumber() <= seqNum)) {
2606                                 abort.setDead();
2607                                 i.remove();
2608
2609                                 if (abort.getTransactionArbitrator() == localMachineId) {
2610                                         liveAbortsGeneratedByLocal.remove(abort.getArbitratorLocalSequenceNumber());
2611                                 }
2612                         }
2613                 }
2614
2615
2616
2617                 if (machineId == localMachineId) {
2618                         // Our own messages are immediately dead.
2619                         if (liveness instanceof LastMessage) {
2620                                 ((LastMessage)liveness).setDead();
2621                         } else if (liveness instanceof Slot) {
2622                                 ((Slot)liveness).setDead();
2623                         } else {
2624                                 throw new Error("Unrecognized type");
2625                         }
2626                 }
2627
2628                 // Get the old last message for this device
2629                 Pair<Long, Liveness> lastMessageEntry = lastMessageTable.put(machineId, new Pair<Long, Liveness>(seqNum, liveness));
2630                 if (lastMessageEntry == null) {
2631                         // If no last message then there is nothing else to process
2632                         return;
2633                 }
2634
2635                 long lastMessageSeqNum = lastMessageEntry.getFirst();
2636                 Liveness lastEntry = lastMessageEntry.getSecond();
2637
2638                 // If it is not our machine Id since we already set ours to dead
2639                 if (machineId != localMachineId) {
2640                         if (lastEntry instanceof LastMessage) {
2641                                 ((LastMessage)lastEntry).setDead();
2642                         } else if (lastEntry instanceof Slot) {
2643                                 ((Slot)lastEntry).setDead();
2644                         } else {
2645                                 throw new Error("Unrecognized type");
2646                         }
2647                 }
2648
2649                 // Make sure the server is not playing any games
2650                 if (machineId == localMachineId) {
2651
2652                         if (hadPartialSendToServer) {
2653                                 // We were not making any updates and we had a machine mismatch
2654                                 if (lastMessageSeqNum > seqNum && !acceptUpdatesToLocal) {
2655                                         throw new Error("Server Error: Mismatch on local machine sequence number, needed at least: " +  lastMessageSeqNum  + " got: " + seqNum);
2656                                 }
2657
2658                         } else {
2659                                 // We were not making any updates and we had a machine mismatch
2660                                 if (lastMessageSeqNum != seqNum && !acceptUpdatesToLocal) {
2661                                         throw new Error("Server Error: Mismatch on local machine sequence number, needed: " +  lastMessageSeqNum + " got: " + seqNum);
2662                                 }
2663                         }
2664                 } else {
2665                         if (lastMessageSeqNum > seqNum) {
2666                                 throw new Error("Server Error: Rollback on remote machine sequence number");
2667                         }
2668                 }
2669         }
2670
2671         /**
2672          * Add a rejected message entry to the watch set to keep track of which clients have seen that
2673          * rejected message entry and which have not.
2674          */
2675         private void addWatchList(long machineId, RejectedMessage entry) {
2676                 HashSet<RejectedMessage> entries = rejectedMessageWatchListTable.get(machineId);
2677                 if (entries == null) {
2678                         // There is no set for this machine ID yet so create one
2679                         entries = new HashSet<RejectedMessage>();
2680                         rejectedMessageWatchListTable.put(machineId, entries);
2681                 }
2682                 entries.add(entry);
2683         }