2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #include <linux/device.h>
16 #include <linux/err.h>
17 #include <linux/errno.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/memblock.h>
23 #include <linux/rslib.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <linux/pstore_ram.h>
29 struct persistent_ram_buffer {
36 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
38 static __initdata LIST_HEAD(persistent_ram_list);
40 static inline size_t buffer_size(struct persistent_ram_zone *prz)
42 return atomic_read(&prz->buffer->size);
45 static inline size_t buffer_start(struct persistent_ram_zone *prz)
47 return atomic_read(&prz->buffer->start);
50 /* increase and wrap the start pointer, returning the old value */
51 static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
57 old = atomic_read(&prz->buffer->start);
59 while (unlikely(new > prz->buffer_size))
60 new -= prz->buffer_size;
61 } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old);
66 /* increase the size counter until it hits the max size */
67 static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
72 if (atomic_read(&prz->buffer->size) == prz->buffer_size)
76 old = atomic_read(&prz->buffer->size);
78 if (new > prz->buffer_size)
79 new = prz->buffer_size;
80 } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old);
83 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,
84 uint8_t *data, size_t len, uint8_t *ecc)
87 uint16_t par[prz->ecc_size];
89 /* Initialize the parity buffer */
90 memset(par, 0, sizeof(par));
91 encode_rs8(prz->rs_decoder, data, len, par, 0);
92 for (i = 0; i < prz->ecc_size; i++)
96 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,
97 void *data, size_t len, uint8_t *ecc)
100 uint16_t par[prz->ecc_size];
102 for (i = 0; i < prz->ecc_size; i++)
104 return decode_rs8(prz->rs_decoder, data, par, len,
105 NULL, 0, NULL, 0, NULL);
108 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,
109 unsigned int start, unsigned int count)
111 struct persistent_ram_buffer *buffer = prz->buffer;
112 uint8_t *buffer_end = buffer->data + prz->buffer_size;
115 int ecc_block_size = prz->ecc_block_size;
116 int ecc_size = prz->ecc_size;
117 int size = prz->ecc_block_size;
122 block = buffer->data + (start & ~(ecc_block_size - 1));
123 par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size;
126 if (block + ecc_block_size > buffer_end)
127 size = buffer_end - block;
128 persistent_ram_encode_rs8(prz, block, size, par);
129 block += ecc_block_size;
131 } while (block < buffer->data + start + count);
134 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)
136 struct persistent_ram_buffer *buffer = prz->buffer;
141 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),
145 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)
147 struct persistent_ram_buffer *buffer = prz->buffer;
154 block = buffer->data;
155 par = prz->par_buffer;
156 while (block < buffer->data + buffer_size(prz)) {
158 int size = prz->ecc_block_size;
159 if (block + size > buffer->data + prz->buffer_size)
160 size = buffer->data + prz->buffer_size - block;
161 numerr = persistent_ram_decode_rs8(prz, block, size, par);
163 pr_devel("persistent_ram: error in block %p, %d\n",
165 prz->corrected_bytes += numerr;
166 } else if (numerr < 0) {
167 pr_devel("persistent_ram: uncorrectable error in block %p\n",
171 block += prz->ecc_block_size;
172 par += prz->ecc_size;
176 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,
180 struct persistent_ram_buffer *buffer = prz->buffer;
186 prz->ecc_block_size = 128;
188 prz->ecc_symsize = 8;
189 prz->ecc_poly = 0x11d;
191 ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size);
192 prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size;
194 if (prz->buffer_size > buffer_size) {
195 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n",
196 buffer_size, prz->buffer_size);
200 prz->par_buffer = buffer->data + prz->buffer_size;
201 prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size;
204 * first consecutive root is 0
205 * primitive element to generate roots = 1
207 prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1,
209 if (prz->rs_decoder == NULL) {
210 pr_info("persistent_ram: init_rs failed\n");
214 prz->corrected_bytes = 0;
217 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),
220 pr_info("persistent_ram: error in header, %d\n", numerr);
221 prz->corrected_bytes += numerr;
222 } else if (numerr < 0) {
223 pr_info("persistent_ram: uncorrectable error in header\n");
230 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,
231 char *str, size_t len)
235 if (prz->corrected_bytes || prz->bad_blocks)
236 ret = snprintf(str, len, ""
237 "\n%d Corrected bytes, %d unrecoverable blocks\n",
238 prz->corrected_bytes, prz->bad_blocks);
240 ret = snprintf(str, len, "\nNo errors detected\n");
245 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,
246 const void *s, unsigned int start, unsigned int count)
248 struct persistent_ram_buffer *buffer = prz->buffer;
249 memcpy(buffer->data + start, s, count);
250 persistent_ram_update_ecc(prz, start, count);
253 void persistent_ram_save_old(struct persistent_ram_zone *prz)
255 struct persistent_ram_buffer *buffer = prz->buffer;
256 size_t size = buffer_size(prz);
257 size_t start = buffer_start(prz);
263 persistent_ram_ecc_old(prz);
264 prz->old_log = kmalloc(size, GFP_KERNEL);
267 pr_err("persistent_ram: failed to allocate buffer\n");
271 prz->old_log_size = size;
272 memcpy(prz->old_log, &buffer->data[start], size - start);
273 memcpy(prz->old_log + size - start, &buffer->data[0], start);
276 int notrace persistent_ram_write(struct persistent_ram_zone *prz,
277 const void *s, unsigned int count)
283 if (unlikely(c > prz->buffer_size)) {
284 s += c - prz->buffer_size;
285 c = prz->buffer_size;
288 buffer_size_add(prz, c);
290 start = buffer_start_add(prz, c);
292 rem = prz->buffer_size - start;
293 if (unlikely(rem < c)) {
294 persistent_ram_update(prz, s, start, rem);
299 persistent_ram_update(prz, s, start, c);
301 persistent_ram_update_header_ecc(prz);
306 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)
308 return prz->old_log_size;
311 void *persistent_ram_old(struct persistent_ram_zone *prz)
316 void persistent_ram_free_old(struct persistent_ram_zone *prz)
320 prz->old_log_size = 0;
323 static void *persistent_ram_vmap(phys_addr_t start, size_t size)
326 phys_addr_t page_start;
327 unsigned int page_count;
332 page_start = start - offset_in_page(start);
333 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE);
335 prot = pgprot_noncached(PAGE_KERNEL);
337 pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL);
339 pr_err("%s: Failed to allocate array for %u pages\n", __func__,
344 for (i = 0; i < page_count; i++) {
345 phys_addr_t addr = page_start + i * PAGE_SIZE;
346 pages[i] = pfn_to_page(addr >> PAGE_SHIFT);
348 vaddr = vmap(pages, page_count, VM_MAP, prot);
354 static void *persistent_ram_iomap(phys_addr_t start, size_t size)
356 if (!request_mem_region(start, size, "persistent_ram")) {
357 pr_err("request mem region (0x%llx@0x%llx) failed\n",
358 (unsigned long long)size, (unsigned long long)start);
362 return ioremap(start, size);
365 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
366 struct persistent_ram_zone *prz)
371 if (pfn_valid(start >> PAGE_SHIFT))
372 prz->vaddr = persistent_ram_vmap(start, size);
374 prz->vaddr = persistent_ram_iomap(start, size);
377 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__,
378 (unsigned long long)size, (unsigned long long)start);
382 prz->buffer = prz->vaddr + offset_in_page(start);
383 prz->buffer_size = size - sizeof(struct persistent_ram_buffer);
388 static int __init persistent_ram_post_init(struct persistent_ram_zone *prz, bool ecc)
394 ret = persistent_ram_init_ecc(prz, prz->buffer_size);
398 if (prz->buffer->sig == PERSISTENT_RAM_SIG) {
399 if (buffer_size(prz) > prz->buffer_size ||
400 buffer_start(prz) > buffer_size(prz))
401 pr_info("persistent_ram: found existing invalid buffer,"
402 " size %zu, start %zu\n",
403 buffer_size(prz), buffer_start(prz));
405 pr_info("persistent_ram: found existing buffer,"
406 " size %zu, start %zu\n",
407 buffer_size(prz), buffer_start(prz));
408 persistent_ram_save_old(prz);
412 pr_info("persistent_ram: no valid data in buffer"
413 " (sig = 0x%08x)\n", prz->buffer->sig);
416 prz->buffer->sig = PERSISTENT_RAM_SIG;
417 atomic_set(&prz->buffer->start, 0);
418 atomic_set(&prz->buffer->size, 0);
423 void persistent_ram_free(struct persistent_ram_zone *prz)
425 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {
429 release_mem_region(prz->paddr, prz->size);
431 persistent_ram_free_old(prz);
435 struct persistent_ram_zone * __init persistent_ram_new(phys_addr_t start,
439 struct persistent_ram_zone *prz;
442 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
444 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
448 ret = persistent_ram_buffer_map(start, size, prz);
452 persistent_ram_post_init(prz, ecc);
453 persistent_ram_update_header_ecc(prz);
462 static int __init persistent_ram_buffer_init(const char *name,
463 struct persistent_ram_zone *prz)
466 struct persistent_ram *ram;
467 struct persistent_ram_descriptor *desc;
470 list_for_each_entry(ram, &persistent_ram_list, node) {
472 for (i = 0; i < ram->num_descs; i++) {
473 desc = &ram->descs[i];
474 if (!strcmp(desc->name, name))
475 return persistent_ram_buffer_map(start,
485 struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc)
487 struct persistent_ram_zone *prz;
490 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
492 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
496 ret = persistent_ram_buffer_init(dev_name(dev), prz);
498 pr_err("persistent_ram: failed to initialize buffer\n");
502 persistent_ram_post_init(prz, ecc);
510 struct persistent_ram_zone * __init
511 persistent_ram_init_ringbuffer(struct device *dev, bool ecc)
513 return __persistent_ram_init(dev, ecc);
516 int __init persistent_ram_early_init(struct persistent_ram *ram)
520 ret = memblock_reserve(ram->start, ram->size);
522 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n",
523 (long)ram->start, (long)(ram->start + ram->size - 1));
527 list_add_tail(&ram->node, &persistent_ram_list);
529 pr_info("Initialized persistent memory from %08lx-%08lx\n",
530 (long)ram->start, (long)(ram->start + ram->size - 1));