6 from pyoram.storage.block_storage import \
7 BlockStorageTypeFactory
8 from pyoram.encrypted_storage.top_cached_encrypted_heap_storage import \
9 TopCachedEncryptedHeapStorage
10 from pyoram.encrypted_storage.encrypted_block_storage import \
12 from pyoram.encrypted_storage.encrypted_heap_storage import \
14 from pyoram.crypto.aes import AES
16 from six.moves import xrange
18 thisdir = os.path.dirname(os.path.abspath(__file__))
20 class _TestTopCachedEncryptedHeapStorage(object):
29 assert cls._init_kwds is not None
30 assert cls._storage_type is not None
31 assert cls._heap_base is not None
32 assert cls._heap_height is not None
33 fd, cls._dummy_name = tempfile.mkstemp()
36 os.remove(cls._dummy_name)
37 except OSError: # pragma: no cover
38 pass # pragma: no cover
40 cls._blocks_per_bucket = 3
42 ((cls._heap_base**(cls._heap_height+1)) - 1)//(cls._heap_base-1)
43 cls._block_count = cls._bucket_count * \
44 cls._blocks_per_bucket
45 cls._testfname = cls.__name__ + "_testfile.bin"
47 f = EncryptedHeapStorage.setup(
51 heap_base=cls._heap_base,
52 blocks_per_bucket=cls._blocks_per_bucket,
53 storage_type=cls._storage_type,
54 initialize=lambda i: bytes(bytearray([i]) * \
56 cls._blocks_per_bucket),
60 for i in range(cls._bucket_count):
61 data = bytearray([i]) * \
63 cls._blocks_per_bucket
64 cls._buckets.append(data)
67 def tearDownClass(cls):
69 os.remove(cls._testfname)
70 except OSError: # pragma: no cover
71 pass # pragma: no cover
73 os.remove(cls._dummy_name)
74 except OSError: # pragma: no cover
75 pass # pragma: no cover
77 def test_factory(self):
78 kwds = dict(self._init_kwds)
79 kwds['cached_levels'] = 0
80 with EncryptedHeapStorage(
83 storage_type=self._storage_type) as f1:
84 with TopCachedEncryptedHeapStorage(f1, **kwds) as f2:
85 self.assertTrue(f1 is f2)
88 fname = ".".join(self.id().split(".")[1:])
90 fname = os.path.join(thisdir, fname)
91 if os.path.exists(fname):
92 os.remove(fname) # pragma: no cover
95 fsetup = EncryptedHeapStorage.setup(
99 heap_base=self._heap_base,
100 storage_type=self._storage_type,
101 blocks_per_bucket=blocks_per_bucket)
103 self.assertEqual(type(fsetup.raw_storage),
104 BlockStorageTypeFactory(self._storage_type))
105 with open(fname, 'rb') as f:
109 TopCachedEncryptedHeapStorage.compute_storage_size(
112 heap_base=self._heap_base,
113 blocks_per_bucket=blocks_per_bucket))
116 TopCachedEncryptedHeapStorage.compute_storage_size(
119 heap_base=self._heap_base,
120 blocks_per_bucket=blocks_per_bucket,
123 with TopCachedEncryptedHeapStorage(
124 EncryptedHeapStorage(
127 storage_type=self._storage_type),
128 **self._init_kwds) as f:
129 self.assertEqual(f.header_data, bytes())
130 self.assertEqual(fsetup.header_data, bytes())
131 self.assertEqual(f.key, fsetup.key)
132 self.assertEqual(f.blocks_per_bucket,
134 self.assertEqual(fsetup.blocks_per_bucket,
136 self.assertEqual(f.bucket_count,
137 (self._heap_base**(self._heap_height+1) - 1)//(self._heap_base-1))
138 self.assertEqual(fsetup.bucket_count,
139 (self._heap_base**(self._heap_height+1) - 1)//(self._heap_base-1))
140 self.assertEqual(f.bucket_size,
141 bsize * blocks_per_bucket)
142 self.assertEqual(fsetup.bucket_size,
143 bsize * blocks_per_bucket)
144 self.assertEqual(f.storage_name, fname)
145 self.assertEqual(fsetup.storage_name, fname)
148 def test_setup_withdata(self):
149 fname = ".".join(self.id().split(".")[1:])
151 fname = os.path.join(thisdir, fname)
152 if os.path.exists(fname):
153 os.remove(fname) # pragma: no cover
155 blocks_per_bucket = 1
156 header_data = bytes(bytearray([0,1,2]))
157 fsetup = EncryptedHeapStorage.setup(
161 heap_base=self._heap_base,
162 storage_type=self._storage_type,
163 blocks_per_bucket=blocks_per_bucket,
164 header_data=header_data)
166 self.assertEqual(type(fsetup.raw_storage),
167 BlockStorageTypeFactory(self._storage_type))
168 with open(fname, 'rb') as f:
172 TopCachedEncryptedHeapStorage.compute_storage_size(
175 heap_base=self._heap_base,
176 header_data=header_data))
177 self.assertTrue(len(header_data) > 0)
179 TopCachedEncryptedHeapStorage.compute_storage_size(
182 heap_base=self._heap_base,
183 storage_type=self._storage_type) <
184 TopCachedEncryptedHeapStorage.compute_storage_size(
187 heap_base=self._heap_base,
188 storage_type=self._storage_type,
189 header_data=header_data),
193 TopCachedEncryptedHeapStorage.compute_storage_size(
196 heap_base=self._heap_base,
197 storage_type=self._storage_type,
198 header_data=header_data,
201 with TopCachedEncryptedHeapStorage(
202 EncryptedHeapStorage(
205 storage_type=self._storage_type),
206 **self._init_kwds) as f:
207 self.assertEqual(f.header_data, header_data)
208 self.assertEqual(fsetup.header_data, header_data)
209 self.assertEqual(f.key, fsetup.key)
210 self.assertEqual(f.blocks_per_bucket,
212 self.assertEqual(fsetup.blocks_per_bucket,
214 self.assertEqual(f.bucket_count,
215 (self._heap_base**(self._heap_height+1) - 1)//(self._heap_base-1))
216 self.assertEqual(fsetup.bucket_count,
217 (self._heap_base**(self._heap_height+1) - 1)//(self._heap_base-1))
218 self.assertEqual(f.bucket_size,
219 bsize * blocks_per_bucket)
220 self.assertEqual(fsetup.bucket_size,
221 bsize * blocks_per_bucket)
222 self.assertEqual(f.storage_name, fname)
223 self.assertEqual(fsetup.storage_name, fname)
226 def test_init_exists(self):
227 self.assertEqual(os.path.exists(self._testfname), True)
228 with EncryptedBlockStorage(self._testfname,
230 storage_type=self._storage_type) as f:
231 databefore = f.read_blocks(list(range(f.block_count)))
232 with self.assertRaises(ValueError):
233 with EncryptedBlockStorage(self._testfname,
235 storage_type=self._storage_type) as fb:
236 with TopCachedEncryptedHeapStorage(
237 EncryptedHeapStorage(fb, key=self._key),
238 **self._init_kwds) as f:
239 pass # pragma: no cover
240 with TopCachedEncryptedHeapStorage(
241 EncryptedHeapStorage(
244 storage_type=self._storage_type),
245 **self._init_kwds) as f:
246 self.assertEqual(f.key, self._key)
247 self.assertEqual(f.bucket_size,
249 self._blocks_per_bucket)
250 self.assertEqual(f.bucket_count,
252 self.assertEqual(f.storage_name, self._testfname)
253 self.assertEqual(f.header_data, bytes())
254 self.assertEqual(os.path.exists(self._testfname), True)
255 with TopCachedEncryptedHeapStorage(
256 EncryptedHeapStorage(
259 storage_type=self._storage_type),
260 **self._init_kwds) as f:
261 dataafter = f.bucket_storage.read_blocks(
262 list(range(f.bucket_storage.block_count)))
263 self.assertEqual(databefore, dataafter)
265 def test_read_path(self):
266 with TopCachedEncryptedHeapStorage(
267 EncryptedHeapStorage(
270 storage_type=self._storage_type),
271 **self._init_kwds) as f:
272 self.assertEqual(f.bytes_sent, 0)
273 self.assertEqual(f.bytes_received, 0)
276 f.virtual_heap.first_bucket_at_level(0), 0)
278 f.virtual_heap.last_leaf_bucket(), 0)
280 for b in range(f.virtual_heap.first_bucket_at_level(0),
281 f.virtual_heap.last_leaf_bucket()+1):
282 full_bucket_path = f.virtual_heap.Node(b).\
283 bucket_path_from_root()
284 all_level_starts = list(range(len(full_bucket_path)+1))
285 for level_start in all_level_starts:
286 data = f.read_path(b, level_start=level_start)
287 bucket_path = full_bucket_path[level_start:]
289 if len(full_bucket_path) <= f._external_level:
291 elif level_start >= f._external_level:
292 total_buckets += len(bucket_path)
294 total_buckets += len(full_bucket_path[f._external_level:])
296 self.assertEqual(f.virtual_heap.Node(b).level+1-level_start,
298 for i, bucket in zip(bucket_path, data):
299 self.assertEqual(list(bytearray(bucket)),
300 list(self._buckets[i]))
302 self.assertEqual(f.bytes_sent, 0)
303 self.assertEqual(f.bytes_received,
304 total_buckets*f.bucket_storage._storage.block_size)
306 def test_write_path(self):
307 data = [bytearray([self._bucket_count]) * \
309 self._blocks_per_bucket
310 for i in xrange(self._block_count)]
311 with TopCachedEncryptedHeapStorage(
312 EncryptedHeapStorage(
315 storage_type=self._storage_type),
316 **self._init_kwds) as f:
317 self.assertEqual(f.bytes_sent, 0)
318 self.assertEqual(f.bytes_received, 0)
321 f.virtual_heap.first_bucket_at_level(0), 0)
323 f.virtual_heap.last_leaf_bucket(), 0)
324 all_buckets = list(range(f.virtual_heap.first_bucket_at_level(0),
325 f.virtual_heap.last_leaf_bucket()+1))
326 random.shuffle(all_buckets)
327 total_read_buckets = 0
328 total_write_buckets = 0
329 for b in all_buckets:
330 full_bucket_path = f.virtual_heap.Node(b).\
331 bucket_path_from_root()
332 all_level_starts = list(range(len(full_bucket_path)+1))
333 random.shuffle(all_level_starts)
334 for level_start in all_level_starts:
335 orig = f.read_path(b, level_start=level_start)
336 bucket_path = full_bucket_path[level_start:]
338 if len(full_bucket_path) <= f._external_level:
340 elif level_start >= f._external_level:
341 total_read_buckets += len(bucket_path)
343 total_read_buckets += len(full_bucket_path[f._external_level:])
345 if level_start != len(full_bucket_path):
346 self.assertNotEqual(len(bucket_path), 0)
347 self.assertEqual(f.virtual_heap.Node(b).level+1-level_start,
349 self.assertEqual(len(orig), len(bucket_path))
351 for i, bucket in zip(bucket_path, orig):
352 self.assertEqual(list(bytearray(bucket)),
353 list(self._buckets[i]))
355 f.write_path(b, [bytes(data[i])
356 for i in bucket_path],
357 level_start=level_start)
358 if len(full_bucket_path) <= f._external_level:
360 elif level_start >= f._external_level:
361 total_write_buckets += len(bucket_path)
363 total_write_buckets += len(full_bucket_path[f._external_level:])
365 new = f.read_path(b, level_start=level_start)
366 if len(full_bucket_path) <= f._external_level:
368 elif level_start >= f._external_level:
369 total_read_buckets += len(bucket_path)
371 total_read_buckets += len(full_bucket_path[f._external_level:])
373 self.assertEqual(len(new), len(bucket_path))
374 for i, bucket in zip(bucket_path, new):
375 self.assertEqual(list(bytearray(bucket)),
378 f.write_path(b, [bytes(self._buckets[i])
379 for i in bucket_path],
380 level_start=level_start)
381 if len(full_bucket_path) <= f._external_level:
383 elif level_start >= f._external_level:
384 total_write_buckets += len(bucket_path)
386 total_write_buckets += len(full_bucket_path[f._external_level:])
389 orig = f.read_path(b, level_start=level_start)
390 if len(full_bucket_path) <= f._external_level:
392 elif level_start >= f._external_level:
393 total_read_buckets += len(bucket_path)
395 total_read_buckets += len(full_bucket_path[f._external_level:])
397 self.assertEqual(len(orig), len(bucket_path))
398 for i, bucket in zip(bucket_path, orig):
399 self.assertEqual(list(bytearray(bucket)),
400 list(self._buckets[i]))
402 full_orig = f.read_path(b)
403 if len(full_bucket_path) <= f._external_level:
406 total_read_buckets += len(full_bucket_path[f._external_level:])
408 for i, bucket in zip(full_bucket_path, full_orig):
409 self.assertEqual(list(bytearray(bucket)),
410 list(self._buckets[i]))
411 for c in xrange(self._heap_base):
412 cn = f.virtual_heap.Node(b).child_node(c)
413 if not f.virtual_heap.is_nil_node(cn):
415 bucket_path = f.virtual_heap.Node(cb).\
416 bucket_path_from_root()
417 orig = f.read_path(cb)
418 if len(bucket_path) <= f._external_level:
421 total_read_buckets += len(bucket_path[f._external_level:])
422 self.assertEqual(len(orig), len(bucket_path))
423 for i, bucket in zip(bucket_path, orig):
424 self.assertEqual(list(bytearray(bucket)),
425 list(self._buckets[i]))
427 self.assertEqual(f.bytes_sent,
428 total_write_buckets*f.bucket_storage._storage.block_size)
429 self.assertEqual(f.bytes_received,
430 total_read_buckets*f.bucket_storage._storage.block_size)
432 def test_update_header_data(self):
433 fname = ".".join(self.id().split(".")[1:])
435 fname = os.path.join(thisdir, fname)
436 if os.path.exists(fname):
437 os.remove(fname) # pragma: no cover
439 blocks_per_bucket = 1
440 header_data = bytes(bytearray([0,1,2]))
441 fsetup = EncryptedHeapStorage.setup(
445 heap_base=self._heap_base,
446 blocks_per_bucket=blocks_per_bucket,
447 header_data=header_data)
449 new_header_data = bytes(bytearray([1,1,1]))
450 with TopCachedEncryptedHeapStorage(
451 EncryptedHeapStorage(
454 storage_type=self._storage_type),
455 **self._init_kwds) as f:
456 self.assertEqual(f.header_data, header_data)
457 f.update_header_data(new_header_data)
458 self.assertEqual(f.header_data, new_header_data)
459 with TopCachedEncryptedHeapStorage(
460 EncryptedHeapStorage(
463 storage_type=self._storage_type),
464 **self._init_kwds) as f:
465 self.assertEqual(f.header_data, new_header_data)
466 with self.assertRaises(ValueError):
467 with TopCachedEncryptedHeapStorage(
468 EncryptedHeapStorage(
471 storage_type=self._storage_type),
472 **self._init_kwds) as f:
473 f.update_header_data(bytes(bytearray([1,1])))
474 with self.assertRaises(ValueError):
475 with TopCachedEncryptedHeapStorage(
476 EncryptedHeapStorage(
479 storage_type=self._storage_type),
480 **self._init_kwds) as f:
481 f.update_header_data(bytes(bytearray([1,1,1,1])))
482 with TopCachedEncryptedHeapStorage(
483 EncryptedHeapStorage(
486 storage_type=self._storage_type),
487 **self._init_kwds) as f:
488 self.assertEqual(f.header_data, new_header_data)
491 def test_locked_flag(self):
492 with TopCachedEncryptedHeapStorage(
493 EncryptedHeapStorage(self._testfname,
495 storage_type=self._storage_type),
496 **self._init_kwds) as f:
497 with self.assertRaises(IOError):
498 with TopCachedEncryptedHeapStorage(
499 EncryptedHeapStorage(self._testfname,
501 storage_type=self._storage_type),
502 **self._init_kwds) as f1:
503 pass # pragma: no cover
504 with self.assertRaises(IOError):
505 with TopCachedEncryptedHeapStorage(
506 EncryptedHeapStorage(self._testfname,
508 storage_type=self._storage_type),
509 **self._init_kwds) as f1:
510 pass # pragma: no cover
511 with TopCachedEncryptedHeapStorage(
512 EncryptedHeapStorage(self._testfname,
514 storage_type=self._storage_type,
516 **self._init_kwds) as f1:
518 with self.assertRaises(IOError):
519 with TopCachedEncryptedHeapStorage(
520 EncryptedHeapStorage(self._testfname,
522 storage_type=self._storage_type),
523 **self._init_kwds) as f1:
524 pass # pragma: no cover
525 with TopCachedEncryptedHeapStorage(
526 EncryptedHeapStorage(self._testfname,
528 storage_type=self._storage_type,
530 **self._init_kwds) as f1:
532 with TopCachedEncryptedHeapStorage(
533 EncryptedHeapStorage(self._testfname,
535 storage_type=self._storage_type,
537 **self._init_kwds) as f1:
539 with TopCachedEncryptedHeapStorage(
540 EncryptedHeapStorage(self._testfname,
542 storage_type=self._storage_type),
543 **self._init_kwds) as f:
546 def test_cache_size(self):
547 with TopCachedEncryptedHeapStorage(
548 EncryptedHeapStorage(self._testfname,
550 storage_type=self._storage_type),
551 **self._init_kwds) as f:
552 num_cached_levels = self._init_kwds.get('cached_levels', 1)
553 if num_cached_levels < 0:
554 num_cached_levels = f.virtual_heap.levels
555 cache_bucket_count = 0
556 for l in xrange(num_cached_levels):
557 if l <= f.virtual_heap.last_level:
558 cache_bucket_count += f.virtual_heap.bucket_count_at_level(l)
559 self.assertEqual(cache_bucket_count > 0, True)
560 self.assertEqual(len(f.cached_bucket_data),
561 cache_bucket_count * f.bucket_size)
563 self.assertEqual(f.bytes_sent, 0)
564 self.assertEqual(f.bytes_received, 0)
565 self.assertEqual(f._root_device.bytes_sent, 0)
567 f._root_device.bytes_received,
568 cache_bucket_count*f._root_device.bucket_storage._storage.block_size)
570 class TestTopCachedEncryptedHeapStorageCacheMMapDefault(
571 _TestTopCachedEncryptedHeapStorage,
574 _storage_type = 'mmap'
578 class TestTopCachedEncryptedHeapStorageMMapCache1(
579 _TestTopCachedEncryptedHeapStorage,
581 _init_kwds = {'cached_levels': 1}
582 _storage_type = 'mmap'
586 class TestTopCachedEncryptedHeapStorageMMapCache2(
587 _TestTopCachedEncryptedHeapStorage,
589 _init_kwds = {'cached_levels': 2}
590 _storage_type = 'mmap'
594 class TestTopCachedEncryptedHeapStorageMMapCache3(
595 _TestTopCachedEncryptedHeapStorage,
597 _init_kwds = {'cached_levels': 3}
598 _storage_type = 'mmap'
602 class TestTopCachedEncryptedHeapStorageMMapCache4(
603 _TestTopCachedEncryptedHeapStorage,
605 _init_kwds = {'cached_levels': 4}
606 _storage_type = 'mmap'
610 class TestTopCachedEncryptedHeapStorageMMapCache5(
611 _TestTopCachedEncryptedHeapStorage,
613 _init_kwds = {'cached_levels': 5}
614 _storage_type = 'mmap'
618 class TestTopCachedEncryptedHeapStorageCacheFileDefault(
619 _TestTopCachedEncryptedHeapStorage,
622 _storage_type = 'file'
626 class TestTopCachedEncryptedHeapStorageFileCache1(
627 _TestTopCachedEncryptedHeapStorage,
629 _init_kwds = {'cached_levels': 1}
630 _storage_type = 'file'
634 class TestTopCachedEncryptedHeapStorageFileCache2(
635 _TestTopCachedEncryptedHeapStorage,
637 _init_kwds = {'cached_levels': 2}
638 _storage_type = 'file'
642 class TestTopCachedEncryptedHeapStorageFileCache3(
643 _TestTopCachedEncryptedHeapStorage,
645 _init_kwds = {'cached_levels': 3}
646 _storage_type = 'file'
650 class TestTopCachedEncryptedHeapStorageFileCache4(
651 _TestTopCachedEncryptedHeapStorage,
653 _init_kwds = {'cached_levels': 4}
654 _storage_type = 'file'
658 class TestTopCachedEncryptedHeapStorageFileCache5(
659 _TestTopCachedEncryptedHeapStorage,
661 _init_kwds = {'cached_levels': 5}
662 _storage_type = 'file'
666 class TestTopCachedEncryptedHeapStorageFileCacheBigConcurrency0(
667 _TestTopCachedEncryptedHeapStorage,
669 _init_kwds = {'cached_levels': 20,
670 'concurrency_level': 0}
671 _storage_type = 'file'
675 class TestTopCachedEncryptedHeapStorageFileCache6Concurrency1(
676 _TestTopCachedEncryptedHeapStorage,
678 _init_kwds = {'cached_levels': 6,
679 'concurrency_level': 1}
680 _storage_type = 'file'
684 class TestTopCachedEncryptedHeapStorageFileCache3ConcurrencyBig(
685 _TestTopCachedEncryptedHeapStorage,
687 _init_kwds = {'cached_levels': 3,
688 'concurrency_level': 20}
689 _storage_type = 'file'
693 class TestTopCachedEncryptedHeapStorageFileCache3Concurrency1Base3(
694 _TestTopCachedEncryptedHeapStorage,
696 _init_kwds = {'cached_levels': 3,
697 'concurrency_level': 3}
698 _storage_type = 'file'
702 class TestTopCachedEncryptedHeapStorageFileCacheAll(
703 _TestTopCachedEncryptedHeapStorage,
705 _init_kwds = {'cached_levels': -1}
706 _storage_type = 'file'
710 if __name__ == "__main__":
711 unittest2.main() # pragma: no cover