2 * xsave/xrstor support.
4 * Author: Suresh Siddha <suresh.b.siddha@intel.com>
6 #include <linux/bootmem.h>
7 #include <linux/compat.h>
9 #ifdef CONFIG_IA32_EMULATION
10 #include <asm/sigcontext32.h>
15 * Supported feature mask by the CPU and the kernel.
19 struct _fpx_sw_bytes fx_sw_reserved;
20 #ifdef CONFIG_IA32_EMULATION
21 struct _fpx_sw_bytes fx_sw_reserved_ia32;
24 static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
27 * If a processor implementation discern that a processor state component is
28 * in its initialized state it may modify the corresponding bit in the
29 * xsave_hdr.xstate_bv as '0', with out modifying the corresponding memory
30 * layout in the case of xsaveopt. While presenting the xstate information to
31 * the user, we always ensure that the memory layout of a feature will be in
32 * the init state if the corresponding header bit is zero. This is to ensure
33 * that the user doesn't see some stale state in the memory layout during
34 * signal handling, debugging etc.
36 void __sanitize_i387_state(struct task_struct *tsk)
39 int feature_bit = 0x2;
40 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
45 BUG_ON(task_thread_info(tsk)->status & TS_USEDFPU);
47 xstate_bv = tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv;
50 * None of the feature bits are in init state. So nothing else
51 * to do for us, as the memory layout is upto date.
53 if ((xstate_bv & pcntxt_mask) == pcntxt_mask)
59 if (!(xstate_bv & XSTATE_FP)) {
66 memset(&fx->st_space[0], 0, 128);
70 * SSE is in init state
72 if (!(xstate_bv & XSTATE_SSE))
73 memset(&fx->xmm_space[0], 0, 256);
75 xstate_bv = (pcntxt_mask & ~xstate_bv) >> 2;
78 * Update all the other memory layouts for which the corresponding
79 * header bit is in the init state.
82 if (xstate_bv & 0x1) {
83 int offset = xstate_offsets[feature_bit];
84 int size = xstate_sizes[feature_bit];
86 memcpy(((void *) fx) + offset,
87 ((void *) init_xstate_buf) + offset,
97 * Check for the presence of extended state information in the
98 * user fpstate pointer in the sigcontext.
100 int check_for_xstate(struct i387_fxsave_struct __user *buf,
101 void __user *fpstate,
102 struct _fpx_sw_bytes *fx_sw_user)
104 int min_xstate_size = sizeof(struct i387_fxsave_struct) +
105 sizeof(struct xsave_hdr_struct);
109 err = __copy_from_user(fx_sw_user, &buf->sw_reserved[0],
110 sizeof(struct _fpx_sw_bytes));
115 * First Magic check failed.
117 if (fx_sw_user->magic1 != FP_XSTATE_MAGIC1)
121 * Check for error scenarios.
123 if (fx_sw_user->xstate_size < min_xstate_size ||
124 fx_sw_user->xstate_size > xstate_size ||
125 fx_sw_user->xstate_size > fx_sw_user->extended_size)
128 err = __get_user(magic2, (__u32 *) (((void *)fpstate) +
129 fx_sw_user->extended_size -
130 FP_XSTATE_MAGIC2_SIZE));
134 * Check for the presence of second magic word at the end of memory
135 * layout. This detects the case where the user just copied the legacy
136 * fpstate layout with out copying the extended state information
137 * in the memory layout.
139 if (magic2 != FP_XSTATE_MAGIC2)
147 * Signal frame handlers.
150 int save_i387_xstate(void __user *buf)
152 struct task_struct *tsk = current;
155 if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size))
158 BUG_ON(sig_xstate_size < xstate_size);
160 if ((unsigned long)buf % 64)
161 printk("save_i387_xstate: bad fpstate %p\n", buf);
166 if (task_thread_info(tsk)->status & TS_USEDFPU) {
168 * Start with clearing the user buffer. This will present a
169 * clean context for the bytes not touched by the fxsave/xsave.
171 err = __clear_user(buf, sig_xstate_size);
176 err = xsave_user(buf);
178 err = fxsave_user(buf);
182 task_thread_info(tsk)->status &= ~TS_USEDFPU;
185 sanitize_i387_state(tsk);
186 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
191 clear_used_math(); /* trigger finit */
194 struct _fpstate __user *fx = buf;
195 struct _xstate __user *x = buf;
198 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved,
199 sizeof(struct _fpx_sw_bytes));
201 err |= __put_user(FP_XSTATE_MAGIC2,
202 (__u32 __user *) (buf + sig_xstate_size
203 - FP_XSTATE_MAGIC2_SIZE));
206 * Read the xstate_bv which we copied (directly from the cpu or
207 * from the state in task struct) to the user buffers and
208 * set the FP/SSE bits.
210 err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
213 * For legacy compatible, we always set FP/SSE bits in the bit
214 * vector while saving the state to the user context. This will
215 * enable us capturing any changes(during sigreturn) to
216 * the FP/SSE bits by the legacy applications which don't touch
217 * xstate_bv in the xsave header.
219 * xsave aware apps can change the xstate_bv in the xsave
220 * header as well as change any contents in the memory layout.
221 * xrestore as part of sigreturn will capture all the changes.
223 xstate_bv |= XSTATE_FPSSE;
225 err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv);
235 * Restore the extended state if present. Otherwise, restore the FP/SSE
238 static int restore_user_xstate(void __user *buf)
240 struct _fpx_sw_bytes fx_sw_user;
244 if (((unsigned long)buf % 64) ||
245 check_for_xstate(buf, buf, &fx_sw_user))
248 mask = fx_sw_user.xstate_bv;
251 * restore the state passed by the user.
253 err = xrestore_user(buf, mask);
258 * init the state skipped by the user.
260 mask = pcntxt_mask & ~mask;
262 xrstor_state(init_xstate_buf, mask);
268 * couldn't find the extended state information in the
269 * memory layout. Restore just the FP/SSE and init all
270 * the other extended state.
272 xrstor_state(init_xstate_buf, pcntxt_mask & ~XSTATE_FPSSE);
273 return fxrstor_checking((__force struct i387_fxsave_struct *)buf);
277 * This restores directly out of user space. Exceptions are handled.
279 int restore_i387_xstate(void __user *buf)
281 struct task_struct *tsk = current;
289 if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
298 if (!(task_thread_info(current)->status & TS_USEDFPU)) {
300 task_thread_info(current)->status |= TS_USEDFPU;
303 err = restore_user_xstate(buf);
305 err = fxrstor_checking((__force struct i387_fxsave_struct *)
309 * Encountered an error while doing the restore from the
310 * user buffer, clear the fpu state.
321 * Prepare the SW reserved portion of the fxsave memory layout, indicating
322 * the presence of the extended state information in the memory layout
323 * pointed by the fpstate pointer in the sigcontext.
324 * This will be saved when ever the FP and extended state context is
325 * saved on the user stack during the signal handler delivery to the user.
327 static void prepare_fx_sw_frame(void)
329 int size_extended = (xstate_size - sizeof(struct i387_fxsave_struct)) +
330 FP_XSTATE_MAGIC2_SIZE;
332 sig_xstate_size = sizeof(struct _fpstate) + size_extended;
334 #ifdef CONFIG_IA32_EMULATION
335 sig_xstate_ia32_size = sizeof(struct _fpstate_ia32) + size_extended;
338 memset(&fx_sw_reserved, 0, sizeof(fx_sw_reserved));
340 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
341 fx_sw_reserved.extended_size = sig_xstate_size;
342 fx_sw_reserved.xstate_bv = pcntxt_mask;
343 fx_sw_reserved.xstate_size = xstate_size;
344 #ifdef CONFIG_IA32_EMULATION
345 memcpy(&fx_sw_reserved_ia32, &fx_sw_reserved,
346 sizeof(struct _fpx_sw_bytes));
347 fx_sw_reserved_ia32.extended_size = sig_xstate_ia32_size;
352 * Represents init state for the supported extended state.
354 struct xsave_struct *init_xstate_buf;
357 unsigned int sig_xstate_size = sizeof(struct _fpstate);
361 * Enable the extended processor state save/restore feature
363 static void __cpuinit __xsave_init(void)
365 set_in_cr4(X86_CR4_OSXSAVE);
368 * Enable all the features that the HW is capable of
369 * and the Linux kernel is aware of.
371 xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
375 * Record the offsets and sizes of different state managed by the xsave
378 static void setup_xstate_features(void)
380 int eax, ebx, ecx, edx, leaf = 0x2;
382 xstate_features = fls64(pcntxt_mask);
383 xstate_offsets = alloc_bootmem(xstate_features * sizeof(int));
384 xstate_sizes = alloc_bootmem(xstate_features * sizeof(int));
387 cpuid_count(0xd, leaf, &eax, &ebx, &ecx, &edx);
392 xstate_offsets[leaf] = ebx;
393 xstate_sizes[leaf] = eax;
400 * setup the xstate image representing the init state
402 static void __init setup_xstate_init(void)
404 setup_xstate_features();
407 * Setup init_xstate_buf to represent the init state of
408 * all the features managed by the xsave
410 init_xstate_buf = alloc_bootmem(xstate_size);
411 init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT;
415 * Init all the features state with header_bv being 0x0
417 xrstor_state(init_xstate_buf, -1);
419 * Dump the init state again. This is to identify the init state
420 * of any feature which is not represented by all zero's.
422 xsave_state(init_xstate_buf, -1);
427 * Enable and initialize the xsave feature.
429 static void __cpuinit xsave_cntxt_init(void)
431 unsigned int eax, ebx, ecx, edx;
433 cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
434 pcntxt_mask = eax + ((u64)edx << 32);
436 if ((pcntxt_mask & XSTATE_FPSSE) != XSTATE_FPSSE) {
437 printk(KERN_ERR "FP/SSE not shown under xsave features 0x%llx\n",
443 * Support only the state known to OS.
445 pcntxt_mask = pcntxt_mask & XCNTXT_MASK;
449 * Recompute the context size for enabled features
451 cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
454 update_regset_xstate_info(xstate_size, pcntxt_mask);
455 prepare_fx_sw_frame();
459 printk(KERN_INFO "xsave/xrstor: enabled xstate_bv 0x%llx, "
461 pcntxt_mask, xstate_size);
464 void __cpuinit xsave_init(void)
470 * Boot processor to setup the FP and extended state context info.
472 if (!smp_processor_id())