1	// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2	/*
3	* Copyright (C) 2017-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4	* Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005
5	* Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All rights reserved.
6	*
7	* This driver produces cryptographically secure pseudorandom data. It is divided
8	* into roughly six sections, each with a section header:
9	*
10	* - Initialization and readiness waiting.
11	* - Fast key erasure RNG, the "crng".
12	* - Entropy accumulation and extraction routines.
13	* - Entropy collection routines.
14	* - Userspace reader/writer interfaces.
15	* - Sysctl interface.
16	*
17	* The high level overview is that there is one input pool, into which
18	* various pieces of data are hashed. Prior to initialization, some of that
19	* data is then "credited" as having a certain number of bits of entropy.
20	* When enough bits of entropy are available, the hash is finalized and
21	* handed as a key to a stream cipher that expands it indefinitely for
22	* various consumers. This key is periodically refreshed as the various
23	* entropy collectors, described below, add data to the input pool.
24	*/
25
26	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
27
28	#include <linux/utsname.h>
29	#include <linux/module.h>
30	#include <linux/kernel.h>
31	#include <linux/major.h>
32	#include <linux/string.h>
33	#include <linux/fcntl.h>
34	#include <linux/slab.h>
35	#include <linux/random.h>
36	#include <linux/poll.h>
37	#include <linux/init.h>
38	#include <linux/fs.h>
39	#include <linux/blkdev.h>
40	#include <linux/interrupt.h>
41	#include <linux/mm.h>
42	#include <linux/nodemask.h>
43	#include <linux/spinlock.h>
44	#include <linux/kthread.h>
45	#include <linux/percpu.h>
46	#include <linux/ptrace.h>
47	#include <linux/workqueue.h>
48	#include <linux/irq.h>
49	#include <linux/ratelimit.h>
50	#include <linux/syscalls.h>
51	#include <linux/completion.h>
52	#include <linux/uuid.h>
53	#include <linux/uaccess.h>
54	#include <linux/suspend.h>
55	#include <linux/siphash.h>
56	#include <linux/sched/isolation.h>
57	#include <crypto/chacha.h>
58	#include <crypto/blake2s.h>
59	#ifdef CONFIG_VDSO_GETRANDOM
60	#include <vdso/getrandom.h>
61	#include <vdso/datapage.h>
62	#include <vdso/vsyscall.h>
63	#endif
64	#include <asm/archrandom.h>
65	#include <asm/processor.h>
66	#include <asm/irq.h>
67	#include <asm/irq_regs.h>
68	#include <asm/io.h>
69
70	/*********************************************************************
71	*
72	* Initialization and readiness waiting.
73	*
74	* Much of the RNG infrastructure is devoted to various dependencies
75	* being able to wait until the RNG has collected enough entropy and
76	* is ready for safe consumption.
77	*
78	*********************************************************************/
79
80	/*
81	* crng_init is protected by base_crng->lock, and only increases
82	* its value (from empty->early->ready).
83	*/
84	static enum {
85	CRNG_EMPTY = 0, /* Little to no entropy collected */
86	CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
87	CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
88	} crng_init __read_mostly = CRNG_EMPTY;
89	static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
90	#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
91	/* Various types of waiters for crng_init->CRNG_READY transition. */
92	static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
93	static struct fasync_struct *fasync;
94	static ATOMIC_NOTIFIER_HEAD(random_ready_notifier);
95
96	/* Control how we warn userspace. */
97	static struct ratelimit_state urandom_warning =
98	RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE);
99	static int ratelimit_disable __read_mostly =
100	IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM);
101	module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
102	MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
103
104	/*
105	* Returns whether or not the input pool has been seeded and thus guaranteed
106	* to supply cryptographically secure random numbers. This applies to: the
107	* /dev/urandom device, the get_random_bytes function, and the get_random_{u8,
108	* u16,u32,u64,long} family of functions.
109	*
110	* Returns: true if the input pool has been seeded.
111	* false if the input pool has not been seeded.
112	*/
113	bool rng_is_initialized(void)
114	{
115	return crng_ready();
116	}
117	EXPORT_SYMBOL(rng_is_initialized);
118
119	static void __cold crng_set_ready(struct work_struct *work)
120	{
121	static_branch_enable(&crng_is_ready);
122	}
123
124	/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
125	static void try_to_generate_entropy(void);
126
127	/*
128	* Wait for the input pool to be seeded and thus guaranteed to supply
129	* cryptographically secure random numbers. This applies to: the /dev/urandom
130	* device, the get_random_bytes function, and the get_random_{u8,u16,u32,u64,
131	* long} family of functions. Using any of these functions without first
132	* calling this function forfeits the guarantee of security.
133	*
134	* Returns: 0 if the input pool has been seeded.
135	* -ERESTARTSYS if the function was interrupted by a signal.
136	*/
137	int wait_for_random_bytes(void)
138	{
139	while (!crng_ready()) {
140	int ret;
141
142	try_to_generate_entropy();
143	ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
144	if (ret)
145	return ret > 0 ? 0 : ret;
146	}
147	return 0;
148	}
149	EXPORT_SYMBOL(wait_for_random_bytes);
150
151	/*
152	* Add a callback function that will be invoked when the crng is initialised,
153	* or immediately if it already has been. Only use this is you are absolutely
154	* sure it is required. Most users should instead be able to test
155	* `rng_is_initialized()` on demand, or make use of `get_random_bytes_wait()`.
156	*/
157	int __cold execute_with_initialized_rng(struct notifier_block *nb)
158	{
159	unsigned long flags;
160	int ret = 0;
161
162	spin_lock_irqsave(&random_ready_notifier.lock, flags);
163	if (crng_ready())
164	nb->notifier_call(nb, 0, NULL);
165	else
166	ret = raw_notifier_chain_register(nh: (struct raw_notifier_head *)&random_ready_notifier.head, nb);
167	spin_unlock_irqrestore(lock: &random_ready_notifier.lock, flags);
168	return ret;
169	}
170
171	#define warn_unseeded_randomness() \
172	if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \
173	printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \
174	__func__, (void *)_RET_IP_, crng_init)
175
176
177	/*********************************************************************
178	*
179	* Fast key erasure RNG, the "crng".
180	*
181	* These functions expand entropy from the entropy extractor into
182	* long streams for external consumption using the "fast key erasure"
183	* RNG described at <https://blog.cr.yp.to/20170723-random.html>.
184	*
185	* There are a few exported interfaces for use by other drivers:
186	*
187	* void get_random_bytes(void *buf, size_t len)
188	* u8 get_random_u8()
189	* u16 get_random_u16()
190	* u32 get_random_u32()
191	* u32 get_random_u32_below(u32 ceil)
192	* u32 get_random_u32_above(u32 floor)
193	* u32 get_random_u32_inclusive(u32 floor, u32 ceil)
194	* u64 get_random_u64()
195	* unsigned long get_random_long()
196	*
197	* These interfaces will return the requested number of random bytes
198	* into the given buffer or as a return value. This is equivalent to
199	* a read from /dev/urandom. The u8, u16, u32, u64, long family of
200	* functions may be higher performance for one-off random integers,
201	* because they do a bit of buffering and do not invoke reseeding
202	* until the buffer is emptied.
203	*
204	*********************************************************************/
205
206	enum {
207	CRNG_RESEED_START_INTERVAL = HZ,
208	CRNG_RESEED_INTERVAL = 60 * HZ
209	};
210
211	static struct {
212	u8 key[CHACHA_KEY_SIZE] __aligned(__alignof__(long));
213	unsigned long generation;
214	spinlock_t lock;
215	} base_crng = {
216	.lock = __SPIN_LOCK_UNLOCKED(base_crng.lock)
217	};
218
219	struct crng {
220	u8 key[CHACHA_KEY_SIZE];
221	unsigned long generation;
222	local_lock_t lock;
223	};
224
225	static DEFINE_PER_CPU(struct crng, crngs) = {
226	.generation = ULONG_MAX,
227	.lock = INIT_LOCAL_LOCK(crngs.lock),
228	};
229
230	/*
231	* Return the interval until the next reseeding, which is normally
232	* CRNG_RESEED_INTERVAL, but during early boot, it is at an interval
233	* proportional to the uptime.
234	*/
235	static unsigned int crng_reseed_interval(void)
236	{
237	static bool early_boot = true;
238
239	if (unlikely(READ_ONCE(early_boot))) {
240	time64_t uptime = ktime_get_seconds();
241	if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
242	WRITE_ONCE(early_boot, false);
243	else
244	return max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
245	(unsigned int)uptime / 2 * HZ);
246	}
247	return CRNG_RESEED_INTERVAL;
248	}
249
250	/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
251	static void extract_entropy(void *buf, size_t len);
252
253	/* This extracts a new crng key from the input pool. */
254	static void crng_reseed(struct work_struct *work)
255	{
256	static DECLARE_DELAYED_WORK(next_reseed, crng_reseed);
257	unsigned long flags;
258	unsigned long next_gen;
259	u8 key[CHACHA_KEY_SIZE];
260
261	/* Immediately schedule the next reseeding, so that it fires sooner rather than later. */
262	if (likely(system_unbound_wq))
263	queue_delayed_work(wq: system_unbound_wq, dwork: &next_reseed, delay: crng_reseed_interval());
264
265	extract_entropy(buf: key, len: sizeof(key));
266
267	/*
268	* We copy the new key into the base_crng, overwriting the old one,
269	* and update the generation counter. We avoid hitting ULONG_MAX,
270	* because the per-cpu crngs are initialized to ULONG_MAX, so this
271	* forces new CPUs that come online to always initialize.
272	*/
273	spin_lock_irqsave(&base_crng.lock, flags);
274	memcpy(base_crng.key, key, sizeof(base_crng.key));
275	next_gen = base_crng.generation + 1;
276	if (next_gen == ULONG_MAX)
277	++next_gen;
278	WRITE_ONCE(base_crng.generation, next_gen);
279	#ifdef CONFIG_VDSO_GETRANDOM
280	/* base_crng.generation's invalid value is ULONG_MAX, while
281	* vdso_k_rng_data->generation's invalid value is 0, so add one to the
282	* former to arrive at the latter. Use smp_store_release so that this
283	* is ordered with the write above to base_crng.generation. Pairs with
284	* the smp_rmb() before the syscall in the vDSO code.
285	*
286	* Cast to unsigned long for 32-bit architectures, since atomic 64-bit
287	* operations are not supported on those architectures. This is safe
288	* because base_crng.generation is a 32-bit value. On big-endian
289	* architectures it will be stored in the upper 32 bits, but that's okay
290	* because the vDSO side only checks whether the value changed, without
291	* actually using or interpreting the value.
292	*/
293	smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
294	#endif
295	if (!static_branch_likely(&crng_is_ready))
296	crng_init = CRNG_READY;
297	spin_unlock_irqrestore(lock: &base_crng.lock, flags);
298	memzero_explicit(s: key, count: sizeof(key));
299	}
300
301	/*
302	* This generates a ChaCha block using the provided key, and then
303	* immediately overwrites that key with half the block. It returns
304	* the resultant ChaCha state to the user, along with the second
305	* half of the block containing 32 bytes of random data that may
306	* be used; random_data_len may not be greater than 32.
307	*
308	* The returned ChaCha state contains within it a copy of the old
309	* key value, at index 4, so the state should always be zeroed out
310	* immediately after using in order to maintain forward secrecy.
311	* If the state cannot be erased in a timely manner, then it is
312	* safer to set the random_data parameter to &chacha_state->x[4]
313	* so that this function overwrites it before returning.
314	*/
315	static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE],
316	struct chacha_state *chacha_state,
317	u8 *random_data, size_t random_data_len)
318	{
319	u8 first_block[CHACHA_BLOCK_SIZE];
320
321	BUG_ON(random_data_len > 32);
322
323	chacha_init_consts(state: chacha_state);
324	memcpy(&chacha_state->x[4], key, CHACHA_KEY_SIZE);
325	memset(&chacha_state->x[12], 0, sizeof(u32) * 4);
326	chacha20_block(state: chacha_state, out: first_block);
327
328	memcpy(key, first_block, CHACHA_KEY_SIZE);
329	memcpy(random_data, first_block + CHACHA_KEY_SIZE, random_data_len);
330	memzero_explicit(s: first_block, count: sizeof(first_block));
331	}
332
333	/*
334	* This function returns a ChaCha state that you may use for generating
335	* random data. It also returns up to 32 bytes on its own of random data
336	* that may be used; random_data_len may not be greater than 32.
337	*/
338	static void crng_make_state(struct chacha_state *chacha_state,
339	u8 *random_data, size_t random_data_len)
340	{
341	unsigned long flags;
342	struct crng *crng;
343
344	BUG_ON(random_data_len > 32);
345
346	/*
347	* For the fast path, we check whether we're ready, unlocked first, and
348	* then re-check once locked later. In the case where we're really not
349	* ready, we do fast key erasure with the base_crng directly, extracting
350	* when crng_init is CRNG_EMPTY.
351	*/
352	if (!crng_ready()) {
353	bool ready;
354
355	spin_lock_irqsave(&base_crng.lock, flags);
356	ready = crng_ready();
357	if (!ready) {
358	if (crng_init == CRNG_EMPTY)
359	extract_entropy(buf: base_crng.key, len: sizeof(base_crng.key));
360	crng_fast_key_erasure(key: base_crng.key, chacha_state,
361	random_data, random_data_len);
362	}
363	spin_unlock_irqrestore(lock: &base_crng.lock, flags);
364	if (!ready)
365	return;
366	}
367
368	local_lock_irqsave(&crngs.lock, flags);
369	crng = raw_cpu_ptr(&crngs);
370
371	/*
372	* If our per-cpu crng is older than the base_crng, then it means
373	* somebody reseeded the base_crng. In that case, we do fast key
374	* erasure on the base_crng, and use its output as the new key
375	* for our per-cpu crng. This brings us up to date with base_crng.
376	*/
377	if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {
378	spin_lock(lock: &base_crng.lock);
379	crng_fast_key_erasure(key: base_crng.key, chacha_state,
380	random_data: crng->key, random_data_len: sizeof(crng->key));
381	crng->generation = base_crng.generation;
382	spin_unlock(lock: &base_crng.lock);
383	}
384
385	/*
386	* Finally, when we've made it this far, our per-cpu crng has an up
387	* to date key, and we can do fast key erasure with it to produce
388	* some random data and a ChaCha state for the caller. All other
389	* branches of this function are "unlikely", so most of the time we
390	* should wind up here immediately.
391	*/
392	crng_fast_key_erasure(key: crng->key, chacha_state, random_data, random_data_len);
393	local_unlock_irqrestore(&crngs.lock, flags);
394	}
395
396	static void _get_random_bytes(void *buf, size_t len)
397	{
398	struct chacha_state chacha_state;
399	u8 tmp[CHACHA_BLOCK_SIZE];
400	size_t first_block_len;
401
402	if (!len)
403	return;
404
405	first_block_len = min_t(size_t, 32, len);
406	crng_make_state(chacha_state: &chacha_state, random_data: buf, random_data_len: first_block_len);
407	len -= first_block_len;
408	buf += first_block_len;
409
410	while (len) {
411	if (len < CHACHA_BLOCK_SIZE) {
412	chacha20_block(state: &chacha_state, out: tmp);
413	memcpy(buf, tmp, len);
414	memzero_explicit(s: tmp, count: sizeof(tmp));
415	break;
416	}
417
418	chacha20_block(state: &chacha_state, out: buf);
419	if (unlikely(chacha_state.x[12] == 0))
420	++chacha_state.x[13];
421	len -= CHACHA_BLOCK_SIZE;
422	buf += CHACHA_BLOCK_SIZE;
423	}
424
425	chacha_zeroize_state(state: &chacha_state);
426	}
427
428	/*
429	* This returns random bytes in arbitrary quantities. The quality of the
430	* random bytes is good as /dev/urandom. In order to ensure that the
431	* randomness provided by this function is okay, the function
432	* wait_for_random_bytes() should be called and return 0 at least once
433	* at any point prior.
434	*/
435	void get_random_bytes(void *buf, size_t len)
436	{
437	warn_unseeded_randomness();
438	_get_random_bytes(buf, len);
439	}
440	EXPORT_SYMBOL(get_random_bytes);
441
442	static ssize_t get_random_bytes_user(struct iov_iter *iter)
443	{
444	struct chacha_state chacha_state;
445	u8 block[CHACHA_BLOCK_SIZE];
446	size_t ret = 0, copied;
447
448	if (unlikely(!iov_iter_count(iter)))
449	return 0;
450
451	/*
452	* Immediately overwrite the ChaCha key at index 4 with random
453	* bytes, in case userspace causes copy_to_iter() below to sleep
454	* forever, so that we still retain forward secrecy in that case.
455	*/
456	crng_make_state(chacha_state: &chacha_state, random_data: (u8 *)&chacha_state.x[4],
457	CHACHA_KEY_SIZE);
458	/*
459	* However, if we're doing a read of len <= 32, we don't need to
460	* use chacha_state after, so we can simply return those bytes to
461	* the user directly.
462	*/
463	if (iov_iter_count(i: iter) <= CHACHA_KEY_SIZE) {
464	ret = copy_to_iter(addr: &chacha_state.x[4], CHACHA_KEY_SIZE, i: iter);
465	goto out_zero_chacha;
466	}
467
468	for (;;) {
469	chacha20_block(state: &chacha_state, out: block);
470	if (unlikely(chacha_state.x[12] == 0))
471	++chacha_state.x[13];
472
473	copied = copy_to_iter(addr: block, bytes: sizeof(block), i: iter);
474	ret += copied;
475	if (!iov_iter_count(i: iter) || copied != sizeof(block))
476	break;
477
478	BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
479	if (ret % PAGE_SIZE == 0) {
480	if (signal_pending(current))
481	break;
482	cond_resched();
483	}
484	}
485
486	memzero_explicit(s: block, count: sizeof(block));
487	out_zero_chacha:
488	chacha_zeroize_state(state: &chacha_state);
489	return ret ? ret : -EFAULT;
490	}
491
492	/*
493	* Batched entropy returns random integers. The quality of the random
494	* number is good as /dev/urandom. In order to ensure that the randomness
495	* provided by this function is okay, the function wait_for_random_bytes()
496	* should be called and return 0 at least once at any point prior.
497	*/
498
499	#define DEFINE_BATCHED_ENTROPY(type) \
500	struct batch_ ##type { \
501	/* \
502	* We make this 1.5x a ChaCha block, so that we get the \
503	* remaining 32 bytes from fast key erasure, plus one full \
504	* block from the detached ChaCha state. We can increase \
505	* the size of this later if needed so long as we keep the \
506	* formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE. \
507	*/ \
508	type entropy[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
509	local_lock_t lock; \
510	unsigned long generation; \
511	unsigned int position; \
512	}; \
513	\
514	static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
515	.lock = INIT_LOCAL_LOCK(batched_entropy_ ##type.lock), \
516	.position = UINT_MAX \
517	}; \
518	\
519	type get_random_ ##type(void) \
520	{ \
521	type ret; \
522	unsigned long flags; \
523	struct batch_ ##type *batch; \
524	unsigned long next_gen; \
525	\
526	warn_unseeded_randomness(); \
527	\
528	if (!crng_ready()) { \
529	_get_random_bytes(&ret, sizeof(ret)); \
530	return ret; \
531	} \
532	\
533	local_lock_irqsave(&batched_entropy_ ##type.lock, flags); \
534	batch = raw_cpu_ptr(&batched_entropy_##type); \
535	\
536	next_gen = READ_ONCE(base_crng.generation); \
537	if (batch->position >= ARRAY_SIZE(batch->entropy) || \
538	next_gen != batch->generation) { \
539	_get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
540	batch->position = 0; \
541	batch->generation = next_gen; \
542	} \
543	\
544	ret = batch->entropy[batch->position]; \
545	batch->entropy[batch->position] = 0; \
546	++batch->position; \
547	local_unlock_irqrestore(&batched_entropy_ ##type.lock, flags); \
548	return ret; \
549	} \
550	EXPORT_SYMBOL(get_random_ ##type);
551
552	DEFINE_BATCHED_ENTROPY(u8)
553	DEFINE_BATCHED_ENTROPY(u16)
554	DEFINE_BATCHED_ENTROPY(u32)
555	DEFINE_BATCHED_ENTROPY(u64)
556
557	u32 __get_random_u32_below(u32 ceil)
558	{
559	/*
560	* This is the slow path for variable ceil. It is still fast, most of
561	* the time, by doing traditional reciprocal multiplication and
562	* opportunistically comparing the lower half to ceil itself, before
563	* falling back to computing a larger bound, and then rejecting samples
564	* whose lower half would indicate a range indivisible by ceil. The use
565	* of `-ceil % ceil` is analogous to `2^32 % ceil`, but is computable
566	* in 32-bits.
567	*/
568	u32 rand = get_random_u32();
569	u64 mult;
570
571	/*
572	* This function is technically undefined for ceil == 0, and in fact
573	* for the non-underscored constant version in the header, we build bug
574	* on that. But for the non-constant case, it's convenient to have that
575	* evaluate to being a straight call to get_random_u32(), so that
576	* get_random_u32_inclusive() can work over its whole range without
577	* undefined behavior.
578	*/
579	if (unlikely(!ceil))
580	return rand;
581
582	mult = (u64)ceil * rand;
583	if (unlikely((u32)mult < ceil)) {
584	u32 bound = -ceil % ceil;
585	while (unlikely((u32)mult < bound))
586	mult = (u64)ceil * get_random_u32();
587	}
588	return mult >> 32;
589	}
590	EXPORT_SYMBOL(__get_random_u32_below);
591
592	#ifdef CONFIG_SMP
593	/*
594	* This function is called when the CPU is coming up, with entry
595	* CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
596	*/
597	int __cold random_prepare_cpu(unsigned int cpu)
598	{
599	/*
600	* When the cpu comes back online, immediately invalidate both
601	* the per-cpu crng and all batches, so that we serve fresh
602	* randomness.
603	*/
604	per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;
605	per_cpu_ptr(&batched_entropy_u8, cpu)->position = UINT_MAX;
606	per_cpu_ptr(&batched_entropy_u16, cpu)->position = UINT_MAX;
607	per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;
608	per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;
609	return 0;
610	}
611	#endif
612
613
614	/**********************************************************************
615	*
616	* Entropy accumulation and extraction routines.
617	*
618	* Callers may add entropy via:
619	*
620	* static void mix_pool_bytes(const void *buf, size_t len)
621	*
622	* After which, if added entropy should be credited:
623	*
624	* static void credit_init_bits(size_t bits)
625	*
626	* Finally, extract entropy via:
627	*
628	* static void extract_entropy(void *buf, size_t len)
629	*
630	**********************************************************************/
631
632	enum {
633	POOL_BITS = BLAKE2S_HASH_SIZE * 8,
634	POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
635	POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
636	};
637
638	static struct {
639	struct blake2s_state hash;
640	spinlock_t lock;
641	unsigned int init_bits;
642	} input_pool = {
643	.hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
644	BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
645	BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 },
646	.hash.outlen = BLAKE2S_HASH_SIZE,
647	.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
648	};
649
650	static void _mix_pool_bytes(const void *buf, size_t len)
651	{
652	blake2s_update(state: &input_pool.hash, in: buf, inlen: len);
653	}
654
655	/*
656	* This function adds bytes into the input pool. It does not
657	* update the initialization bit counter; the caller should call
658	* credit_init_bits if this is appropriate.
659	*/
660	static void mix_pool_bytes(const void *buf, size_t len)
661	{
662	unsigned long flags;
663
664	spin_lock_irqsave(&input_pool.lock, flags);
665	_mix_pool_bytes(buf, len);
666	spin_unlock_irqrestore(lock: &input_pool.lock, flags);
667	}
668
669	/*
670	* This is an HKDF-like construction for using the hashed collected entropy
671	* as a PRF key, that's then expanded block-by-block.
672	*/
673	static void extract_entropy(void *buf, size_t len)
674	{
675	unsigned long flags;
676	u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
677	struct {
678	unsigned long rdseed[32 / sizeof(long)];
679	size_t counter;
680	} block;
681	size_t i, longs;
682
683	for (i = 0; i < ARRAY_SIZE(block.rdseed);) {
684	longs = arch_get_random_seed_longs(v: &block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
685	if (longs) {
686	i += longs;
687	continue;
688	}
689	longs = arch_get_random_longs(v: &block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
690	if (longs) {
691	i += longs;
692	continue;
693	}
694	block.rdseed[i++] = random_get_entropy();
695	}
696
697	spin_lock_irqsave(&input_pool.lock, flags);
698
699	/* seed = HASHPRF(last_key, entropy_input) */
700	blake2s_final(state: &input_pool.hash, out: seed);
701
702	/* next_key = HASHPRF(seed, RDSEED || 0) */
703	block.counter = 0;
704	blake2s(out: next_key, in: (u8 *)&block, key: seed, outlen: sizeof(next_key), inlen: sizeof(block), keylen: sizeof(seed));
705	blake2s_init_key(state: &input_pool.hash, outlen: BLAKE2S_HASH_SIZE, key: next_key, keylen: sizeof(next_key));
706
707	spin_unlock_irqrestore(lock: &input_pool.lock, flags);
708	memzero_explicit(s: next_key, count: sizeof(next_key));
709
710	while (len) {
711	i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
712	/* output = HASHPRF(seed, RDSEED || ++counter) */
713	++block.counter;
714	blake2s(out: buf, in: (u8 *)&block, key: seed, outlen: i, inlen: sizeof(block), keylen: sizeof(seed));
715	len -= i;
716	buf += i;
717	}
718
719	memzero_explicit(s: seed, count: sizeof(seed));
720	memzero_explicit(s: &block, count: sizeof(block));
721	}
722
723	#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
724
725	static void __cold _credit_init_bits(size_t bits)
726	{
727	static DECLARE_WORK(set_ready, crng_set_ready);
728	unsigned int new, orig, add;
729	unsigned long flags;
730	int m;
731
732	if (!bits)
733	return;
734
735	add = min_t(size_t, bits, POOL_BITS);
736
737	orig = READ_ONCE(input_pool.init_bits);
738	do {
739	new = min_t(unsigned int, POOL_BITS, orig + add);
740	} while (!try_cmpxchg(&input_pool.init_bits, &orig, new));
741
742	if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
743	crng_reseed(NULL); /* Sets crng_init to CRNG_READY under base_crng.lock. */
744	if (static_key_initialized && system_unbound_wq)
745	queue_work(wq: system_unbound_wq, work: &set_ready);
746	atomic_notifier_call_chain(nh: &random_ready_notifier, val: 0, NULL);
747	#ifdef CONFIG_VDSO_GETRANDOM
748	WRITE_ONCE(vdso_k_rng_data->is_ready, true);
749	#endif
750	wake_up_interruptible(&crng_init_wait);
751	kill_fasync(&fasync, SIGIO, POLL_IN);
752	pr_notice("crng init done\n");
753	m = ratelimit_state_get_miss(rs: &urandom_warning);
754	if (m)
755	pr_notice("%d urandom warning(s) missed due to ratelimiting\n", m);
756	} else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
757	spin_lock_irqsave(&base_crng.lock, flags);
758	/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
759	if (crng_init == CRNG_EMPTY) {
760	extract_entropy(buf: base_crng.key, len: sizeof(base_crng.key));
761	crng_init = CRNG_EARLY;
762	}
763	spin_unlock_irqrestore(lock: &base_crng.lock, flags);
764	}
765	}
766
767
768	/**********************************************************************
769	*
770	* Entropy collection routines.
771	*
772	* The following exported functions are used for pushing entropy into
773	* the above entropy accumulation routines:
774	*
775	* void add_device_randomness(const void *buf, size_t len);
776	* void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after);
777	* void add_bootloader_randomness(const void *buf, size_t len);
778	* void add_vmfork_randomness(const void *unique_vm_id, size_t len);
779	* void add_interrupt_randomness(int irq);
780	* void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
781	* void add_disk_randomness(struct gendisk *disk);
782	*
783	* add_device_randomness() adds data to the input pool that
784	* is likely to differ between two devices (or possibly even per boot).
785	* This would be things like MAC addresses or serial numbers, or the
786	* read-out of the RTC. This does *not* credit any actual entropy to
787	* the pool, but it initializes the pool to different values for devices
788	* that might otherwise be identical and have very little entropy
789	* available to them (particularly common in the embedded world).
790	*
791	* add_hwgenerator_randomness() is for true hardware RNGs, and will credit
792	* entropy as specified by the caller. If the entropy pool is full it will
793	* block until more entropy is needed.
794	*
795	* add_bootloader_randomness() is called by bootloader drivers, such as EFI
796	* and device tree, and credits its input depending on whether or not the
797	* command line option 'random.trust_bootloader'.
798	*
799	* add_vmfork_randomness() adds a unique (but not necessarily secret) ID
800	* representing the current instance of a VM to the pool, without crediting,
801	* and then force-reseeds the crng so that it takes effect immediately.
802	*
803	* add_interrupt_randomness() uses the interrupt timing as random
804	* inputs to the entropy pool. Using the cycle counters and the irq source
805	* as inputs, it feeds the input pool roughly once a second or after 64
806	* interrupts, crediting 1 bit of entropy for whichever comes first.
807	*
808	* add_input_randomness() uses the input layer interrupt timing, as well
809	* as the event type information from the hardware.
810	*
811	* add_disk_randomness() uses what amounts to the seek time of block
812	* layer request events, on a per-disk_devt basis, as input to the
813	* entropy pool. Note that high-speed solid state drives with very low
814	* seek times do not make for good sources of entropy, as their seek
815	* times are usually fairly consistent.
816	*
817	* The last two routines try to estimate how many bits of entropy
818	* to credit. They do this by keeping track of the first and second
819	* order deltas of the event timings.
820	*
821	**********************************************************************/
822
823	static bool trust_cpu __initdata = true;
824	static bool trust_bootloader __initdata = true;
825	static int __init parse_trust_cpu(char *arg)
826	{
827	return kstrtobool(s: arg, res: &trust_cpu);
828	}
829	static int __init parse_trust_bootloader(char *arg)
830	{
831	return kstrtobool(s: arg, res: &trust_bootloader);
832	}
833	early_param("random.trust_cpu", parse_trust_cpu);
834	early_param("random.trust_bootloader", parse_trust_bootloader);
835
836	static int random_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
837	{
838	unsigned long flags, entropy = random_get_entropy();
839
840	/*
841	* Encode a representation of how long the system has been suspended,
842	* in a way that is distinct from prior system suspends.
843	*/
844	ktime_t stamps[] = { ktime_get(), ktime_get_boottime(), ktime_get_real() };
845
846	spin_lock_irqsave(&input_pool.lock, flags);
847	_mix_pool_bytes(buf: &action, len: sizeof(action));
848	_mix_pool_bytes(buf: stamps, len: sizeof(stamps));
849	_mix_pool_bytes(buf: &entropy, len: sizeof(entropy));
850	spin_unlock_irqrestore(lock: &input_pool.lock, flags);
851
852	if (crng_ready() && (action == PM_RESTORE_PREPARE ||
853	(action == PM_POST_SUSPEND && !IS_ENABLED(CONFIG_PM_AUTOSLEEP) &&
854	!IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)))) {
855	crng_reseed(NULL);
856	pr_notice("crng reseeded on system resumption\n");
857	}
858	return 0;
859	}
860
861	static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification };
862
863	/*
864	* This is called extremely early, before time keeping functionality is
865	* available, but arch randomness is. Interrupts are not yet enabled.
866	*/
867	void __init random_init_early(const char *command_line)
868	{
869	unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)];
870	size_t i, longs, arch_bits;
871
872	#if defined(LATENT_ENTROPY_PLUGIN)
873	static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
874	_mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
875	#endif
876
877	for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) {
878	longs = arch_get_random_seed_longs(v: entropy, ARRAY_SIZE(entropy) - i);
879	if (longs) {
880	_mix_pool_bytes(buf: entropy, len: sizeof(*entropy) * longs);
881	i += longs;
882	continue;
883	}
884	longs = arch_get_random_longs(v: entropy, ARRAY_SIZE(entropy) - i);
885	if (longs) {
886	_mix_pool_bytes(buf: entropy, len: sizeof(*entropy) * longs);
887	i += longs;
888	continue;
889	}
890	arch_bits -= sizeof(*entropy) * 8;
891	++i;
892	}
893
894	_mix_pool_bytes(buf: init_utsname(), len: sizeof(*(init_utsname())));
895	_mix_pool_bytes(buf: command_line, strlen(command_line));
896
897	/* Reseed if already seeded by earlier phases. */
898	if (crng_ready())
899	crng_reseed(NULL);
900	else if (trust_cpu)
901	_credit_init_bits(bits: arch_bits);
902	}
903
904	/*
905	* This is called a little bit after the prior function, and now there is
906	* access to timestamps counters. Interrupts are not yet enabled.
907	*/
908	void __init random_init(void)
909	{
910	unsigned long entropy = random_get_entropy();
911	ktime_t now = ktime_get_real();
912
913	_mix_pool_bytes(buf: &now, len: sizeof(now));
914	_mix_pool_bytes(buf: &entropy, len: sizeof(entropy));
915	add_latent_entropy();
916
917	/*
918	* If we were initialized by the cpu or bootloader before jump labels
919	* or workqueues are initialized, then we should enable the static
920	* branch here, where it's guaranteed that these have been initialized.
921	*/
922	if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY)
923	crng_set_ready(NULL);
924
925	/* Reseed if already seeded by earlier phases. */
926	if (crng_ready())
927	crng_reseed(NULL);
928
929	WARN_ON(register_pm_notifier(&pm_notifier));
930
931	WARN(!entropy, "Missing cycle counter and fallback timer; RNG "
932	"entropy collection will consequently suffer.");
933	}
934
935	/*
936	* Add device- or boot-specific data to the input pool to help
937	* initialize it.
938	*
939	* None of this adds any entropy; it is meant to avoid the problem of
940	* the entropy pool having similar initial state across largely
941	* identical devices.
942	*/
943	void add_device_randomness(const void *buf, size_t len)
944	{
945	unsigned long entropy = random_get_entropy();
946	unsigned long flags;
947
948	spin_lock_irqsave(&input_pool.lock, flags);
949	_mix_pool_bytes(buf: &entropy, len: sizeof(entropy));
950	_mix_pool_bytes(buf, len);
951	spin_unlock_irqrestore(lock: &input_pool.lock, flags);
952	}
953	EXPORT_SYMBOL(add_device_randomness);
954
955	/*
956	* Interface for in-kernel drivers of true hardware RNGs. Those devices
957	* may produce endless random bits, so this function will sleep for
958	* some amount of time after, if the sleep_after parameter is true.
959	*/
960	void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after)
961	{
962	mix_pool_bytes(buf, len);
963	credit_init_bits(entropy);
964
965	/*
966	* Throttle writing to once every reseed interval, unless we're not yet
967	* initialized or no entropy is credited.
968	*/
969	if (sleep_after && !kthread_should_stop() && (crng_ready() || !entropy))
970	schedule_timeout_interruptible(timeout: crng_reseed_interval());
971	}
972	EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
973
974	/*
975	* Handle random seed passed by bootloader, and credit it depending
976	* on the command line option 'random.trust_bootloader'.
977	*/
978	void __init add_bootloader_randomness(const void *buf, size_t len)
979	{
980	mix_pool_bytes(buf, len);
981	if (trust_bootloader)
982	credit_init_bits(len * 8);
983	}
984
985	#if IS_ENABLED(CONFIG_VMGENID)
986	static BLOCKING_NOTIFIER_HEAD(vmfork_chain);
987
988	/*
989	* Handle a new unique VM ID, which is unique, not secret, so we
990	* don't credit it, but we do immediately force a reseed after so
991	* that it's used by the crng posthaste.
992	*/
993	void __cold add_vmfork_randomness(const void *unique_vm_id, size_t len)
994	{
995	add_device_randomness(unique_vm_id, len);
996	if (crng_ready()) {
997	crng_reseed(NULL);
998	pr_notice("crng reseeded due to virtual machine fork\n");
999	}
1000	blocking_notifier_call_chain(nh: &vmfork_chain, val: 0, NULL);
1001	}
1002	#if IS_MODULE(CONFIG_VMGENID)
1003	EXPORT_SYMBOL_GPL(add_vmfork_randomness);
1004	#endif
1005
1006	int __cold register_random_vmfork_notifier(struct notifier_block *nb)
1007	{
1008	return blocking_notifier_chain_register(nh: &vmfork_chain, nb);
1009	}
1010	EXPORT_SYMBOL_GPL(register_random_vmfork_notifier);
1011
1012	int __cold unregister_random_vmfork_notifier(struct notifier_block *nb)
1013	{
1014	return blocking_notifier_chain_unregister(nh: &vmfork_chain, nb);
1015	}
1016	EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier);
1017	#endif
1018
1019	struct fast_pool {
1020	unsigned long pool[4];
1021	unsigned long last;
1022	unsigned int count;
1023	struct timer_list mix;
1024	};
1025
1026	static void mix_interrupt_randomness(struct timer_list *work);
1027
1028	static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
1029	#ifdef CONFIG_64BIT
1030	#define FASTMIX_PERM SIPHASH_PERMUTATION
1031	.pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 },
1032	#else
1033	#define FASTMIX_PERM HSIPHASH_PERMUTATION
1034	.pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 },
1035	#endif
1036	.mix = __TIMER_INITIALIZER(mix_interrupt_randomness, 0)
1037	};
1038
1039	/*
1040	* This is [Half]SipHash-1-x, starting from an empty key. Because
1041	* the key is fixed, it assumes that its inputs are non-malicious,
1042	* and therefore this has no security on its own. s represents the
1043	* four-word SipHash state, while v represents a two-word input.
1044	*/
1045	static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
1046	{
1047	s[3] ^= v1;
1048	FASTMIX_PERM(s[0], s[1], s[2], s[3]);
1049	s[0] ^= v1;
1050	s[3] ^= v2;
1051	FASTMIX_PERM(s[0], s[1], s[2], s[3]);
1052	s[0] ^= v2;
1053	}
1054
1055	#ifdef CONFIG_SMP
1056	/*
1057	* This function is called when the CPU has just come online, with
1058	* entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
1059	*/
1060	int __cold random_online_cpu(unsigned int cpu)
1061	{
1062	/*
1063	* During CPU shutdown and before CPU onlining, add_interrupt_
1064	* randomness() may schedule mix_interrupt_randomness(), and
1065	* set the MIX_INFLIGHT flag. However, because the worker can
1066	* be scheduled on a different CPU during this period, that
1067	* flag will never be cleared. For that reason, we zero out
1068	* the flag here, which runs just after workqueues are onlined
1069	* for the CPU again. This also has the effect of setting the
1070	* irq randomness count to zero so that new accumulated irqs
1071	* are fresh.
1072	*/
1073	per_cpu_ptr(&irq_randomness, cpu)->count = 0;
1074	return 0;
1075	}
1076	#endif
1077
1078	static void mix_interrupt_randomness(struct timer_list *work)
1079	{
1080	struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
1081	/*
1082	* The size of the copied stack pool is explicitly 2 longs so that we
1083	* only ever ingest half of the siphash output each time, retaining
1084	* the other half as the next "key" that carries over. The entropy is
1085	* supposed to be sufficiently dispersed between bits so on average
1086	* we don't wind up "losing" some.
1087	*/
1088	unsigned long pool[2];
1089	unsigned int count;
1090
1091	/* Check to see if we're running on the wrong CPU due to hotplug. */
1092	local_irq_disable();
1093	if (fast_pool != this_cpu_ptr(&irq_randomness)) {
1094	local_irq_enable();
1095	return;
1096	}
1097
1098	/*
1099	* Copy the pool to the stack so that the mixer always has a
1100	* consistent view, before we reenable irqs again.
1101	*/
1102	memcpy(pool, fast_pool->pool, sizeof(pool));
1103	count = fast_pool->count;
1104	fast_pool->count = 0;
1105	fast_pool->last = jiffies;
1106	local_irq_enable();
1107
1108	mix_pool_bytes(buf: pool, len: sizeof(pool));
1109	credit_init_bits(clamp_t(unsigned int, (count & U16_MAX) / 64, 1, sizeof(pool) * 8));
1110
1111	memzero_explicit(s: pool, count: sizeof(pool));
1112	}
1113
1114	void add_interrupt_randomness(int irq)
1115	{
1116	enum { MIX_INFLIGHT = 1U << 31 };
1117	unsigned long entropy = random_get_entropy();
1118	struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
1119	struct pt_regs *regs = get_irq_regs();
1120	unsigned int new_count;
1121
1122	fast_mix(s: fast_pool->pool, v1: entropy,
1123	v2: (regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(y: irq));
1124	new_count = ++fast_pool->count;
1125
1126	if (new_count & MIX_INFLIGHT)
1127	return;
1128
1129	if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ))
1130	return;
1131
1132	fast_pool->count |= MIX_INFLIGHT;
1133	if (!timer_pending(timer: &fast_pool->mix)) {
1134	fast_pool->mix.expires = jiffies;
1135	add_timer_on(timer: &fast_pool->mix, raw_smp_processor_id());
1136	}
1137	}
1138	EXPORT_SYMBOL_GPL(add_interrupt_randomness);
1139
1140	/* There is one of these per entropy source */
1141	struct timer_rand_state {
1142	unsigned long last_time;
1143	long last_delta, last_delta2;
1144	};
1145
1146	/*
1147	* This function adds entropy to the entropy "pool" by using timing
1148	* delays. It uses the timer_rand_state structure to make an estimate
1149	* of how many bits of entropy this call has added to the pool. The
1150	* value "num" is also added to the pool; it should somehow describe
1151	* the type of event that just happened.
1152	*/
1153	static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
1154	{
1155	unsigned long entropy = random_get_entropy(), now = jiffies, flags;
1156	long delta, delta2, delta3;
1157	unsigned int bits;
1158
1159	/*
1160	* If we're in a hard IRQ, add_interrupt_randomness() will be called
1161	* sometime after, so mix into the fast pool.
1162	*/
1163	if (in_hardirq()) {
1164	fast_mix(this_cpu_ptr(&irq_randomness)->pool, v1: entropy, v2: num);
1165	} else {
1166	spin_lock_irqsave(&input_pool.lock, flags);
1167	_mix_pool_bytes(buf: &entropy, len: sizeof(entropy));
1168	_mix_pool_bytes(buf: &num, len: sizeof(num));
1169	spin_unlock_irqrestore(lock: &input_pool.lock, flags);
1170	}
1171
1172	if (crng_ready())
1173	return;
1174
1175	/*
1176	* Calculate number of bits of randomness we probably added.
1177	* We take into account the first, second and third-order deltas
1178	* in order to make our estimate.
1179	*/
1180	delta = now - READ_ONCE(state->last_time);
1181	WRITE_ONCE(state->last_time, now);
1182
1183	delta2 = delta - READ_ONCE(state->last_delta);
1184	WRITE_ONCE(state->last_delta, delta);
1185
1186	delta3 = delta2 - READ_ONCE(state->last_delta2);
1187	WRITE_ONCE(state->last_delta2, delta2);
1188
1189	if (delta < 0)
1190	delta = -delta;
1191	if (delta2 < 0)
1192	delta2 = -delta2;
1193	if (delta3 < 0)
1194	delta3 = -delta3;
1195	if (delta > delta2)
1196	delta = delta2;
1197	if (delta > delta3)
1198	delta = delta3;
1199
1200	/*
1201	* delta is now minimum absolute delta. Round down by 1 bit
1202	* on general principles, and limit entropy estimate to 11 bits.
1203	*/
1204	bits = min(fls(delta >> 1), 11);
1205
1206	/*
1207	* As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
1208	* will run after this, which uses a different crediting scheme of 1 bit
1209	* per every 64 interrupts. In order to let that function do accounting
1210	* close to the one in this function, we credit a full 64/64 bit per bit,
1211	* and then subtract one to account for the extra one added.
1212	*/
1213	if (in_hardirq())
1214	this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
1215	else
1216	_credit_init_bits(bits);
1217	}
1218
1219	void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
1220	{
1221	static unsigned char last_value;
1222	static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
1223
1224	/* Ignore autorepeat and the like. */
1225	if (value == last_value)
1226	return;
1227
1228	last_value = value;
1229	add_timer_randomness(state: &input_timer_state,
1230	num: (type << 4) ^ code ^ (code >> 4) ^ value);
1231	}
1232	EXPORT_SYMBOL_GPL(add_input_randomness);
1233
1234	#ifdef CONFIG_BLOCK
1235	void add_disk_randomness(struct gendisk *disk)
1236	{
1237	if (!disk || !disk->random)
1238	return;
1239	/* First major is 1, so we get >= 0x200 here. */
1240	add_timer_randomness(state: disk->random, num: 0x100 + disk_devt(disk));
1241	}
1242	EXPORT_SYMBOL_GPL(add_disk_randomness);
1243
1244	void __cold rand_initialize_disk(struct gendisk *disk)
1245	{
1246	struct timer_rand_state *state;
1247
1248	/*
1249	* If kzalloc returns null, we just won't use that entropy
1250	* source.
1251	*/
1252	state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
1253	if (state) {
1254	state->last_time = INITIAL_JIFFIES;
1255	disk->random = state;
1256	}
1257	}
1258	#endif
1259
1260	struct entropy_timer_state {
1261	unsigned long entropy;
1262	struct timer_list timer;
1263	atomic_t samples;
1264	unsigned int samples_per_bit;
1265	};
1266
1267	/*
1268	* Each time the timer fires, we expect that we got an unpredictable jump in
1269	* the cycle counter. Even if the timer is running on another CPU, the timer
1270	* activity will be touching the stack of the CPU that is generating entropy.
1271	*
1272	* Note that we don't re-arm the timer in the timer itself - we are happy to be
1273	* scheduled away, since that just makes the load more complex, but we do not
1274	* want the timer to keep ticking unless the entropy loop is running.
1275	*
1276	* So the re-arming always happens in the entropy loop itself.
1277	*/
1278	static void __cold entropy_timer(struct timer_list *timer)
1279	{
1280	struct entropy_timer_state *state = container_of(timer, struct entropy_timer_state, timer);
1281	unsigned long entropy = random_get_entropy();
1282
1283	mix_pool_bytes(buf: &entropy, len: sizeof(entropy));
1284	if (atomic_inc_return(v: &state->samples) % state->samples_per_bit == 0)
1285	credit_init_bits(1);
1286	}
1287
1288	/*
1289	* If we have an actual cycle counter, see if we can generate enough entropy
1290	* with timing noise.
1291	*/
1292	static void __cold try_to_generate_entropy(void)
1293	{
1294	enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 15 };
1295	u8 stack_bytes[sizeof(struct entropy_timer_state) + SMP_CACHE_BYTES - 1];
1296	struct entropy_timer_state *stack = PTR_ALIGN((void *)stack_bytes, SMP_CACHE_BYTES);
1297	unsigned int i, num_different = 0;
1298	unsigned long last = random_get_entropy();
1299	int cpu = -1;
1300
1301	for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) {
1302	stack->entropy = random_get_entropy();
1303	if (stack->entropy != last)
1304	++num_different;
1305	last = stack->entropy;
1306	}
1307	stack->samples_per_bit = DIV_ROUND_UP(NUM_TRIAL_SAMPLES, num_different + 1);
1308	if (stack->samples_per_bit > MAX_SAMPLES_PER_BIT)
1309	return;
1310
1311	atomic_set(v: &stack->samples, i: 0);
1312	timer_setup_on_stack(&stack->timer, entropy_timer, 0);
1313	while (!crng_ready() && !signal_pending(current)) {
1314	/*
1315	* Check !timer_pending() and then ensure that any previous callback has finished
1316	* executing by checking timer_delete_sync_try(), before queueing the next one.
1317	*/
1318	if (!timer_pending(timer: &stack->timer) && timer_delete_sync_try(timer: &stack->timer) >= 0) {
1319	struct cpumask timer_cpus;
1320	unsigned int num_cpus;
1321
1322	/*
1323	* Preemption must be disabled here, both to read the current CPU number
1324	* and to avoid scheduling a timer on a dead CPU.
1325	*/
1326	preempt_disable();
1327
1328	/* Only schedule callbacks on timer CPUs that are online. */
1329	cpumask_and(dstp: &timer_cpus, src1p: housekeeping_cpumask(type: HK_TYPE_TIMER), cpu_online_mask);
1330	num_cpus = cpumask_weight(srcp: &timer_cpus);
1331	/* In very bizarre case of misconfiguration, fallback to all online. */
1332	if (unlikely(num_cpus == 0)) {
1333	timer_cpus = *cpu_online_mask;
1334	num_cpus = cpumask_weight(srcp: &timer_cpus);
1335	}
1336
1337	/* Basic CPU round-robin, which avoids the current CPU. */
1338	do {
1339	cpu = cpumask_next(n: cpu, srcp: &timer_cpus);
1340	if (cpu >= nr_cpu_ids)
1341	cpu = cpumask_first(srcp: &timer_cpus);
1342	} while (cpu == smp_processor_id() && num_cpus > 1);
1343
1344	/* Expiring the timer at `jiffies` means it's the next tick. */
1345	stack->timer.expires = jiffies;
1346
1347	add_timer_on(timer: &stack->timer, cpu);
1348
1349	preempt_enable();
1350	}
1351	mix_pool_bytes(buf: &stack->entropy, len: sizeof(stack->entropy));
1352	schedule();
1353	stack->entropy = random_get_entropy();
1354	}
1355	mix_pool_bytes(buf: &stack->entropy, len: sizeof(stack->entropy));
1356
1357	timer_delete_sync(timer: &stack->timer);
1358	timer_destroy_on_stack(timer: &stack->timer);
1359	}
1360
1361
1362	/**********************************************************************
1363	*
1364	* Userspace reader/writer interfaces.
1365	*
1366	* getrandom(2) is the primary modern interface into the RNG and should
1367	* be used in preference to anything else.
1368	*
1369	* Reading from /dev/random has the same functionality as calling
1370	* getrandom(2) with flags=0. In earlier versions, however, it had
1371	* vastly different semantics and should therefore be avoided, to
1372	* prevent backwards compatibility issues.
1373	*
1374	* Reading from /dev/urandom has the same functionality as calling
1375	* getrandom(2) with flags=GRND_INSECURE. Because it does not block
1376	* waiting for the RNG to be ready, it should not be used.
1377	*
1378	* Writing to either /dev/random or /dev/urandom adds entropy to
1379	* the input pool but does not credit it.
1380	*
1381	* Polling on /dev/random indicates when the RNG is initialized, on
1382	* the read side, and when it wants new entropy, on the write side.
1383	*
1384	* Both /dev/random and /dev/urandom have the same set of ioctls for
1385	* adding entropy, getting the entropy count, zeroing the count, and
1386	* reseeding the crng.
1387	*
1388	**********************************************************************/
1389
1390	SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
1391	{
1392	struct iov_iter iter;
1393	int ret;
1394
1395	if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
1396	return -EINVAL;
1397
1398	/*
1399	* Requesting insecure and blocking randomness at the same time makes
1400	* no sense.
1401	*/
1402	if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
1403	return -EINVAL;
1404
1405	if (!crng_ready() && !(flags & GRND_INSECURE)) {
1406	if (flags & GRND_NONBLOCK)
1407	return -EAGAIN;
1408	ret = wait_for_random_bytes();
1409	if (unlikely(ret))
1410	return ret;
1411	}
1412
1413	ret = import_ubuf(ITER_DEST, buf: ubuf, len, i: &iter);
1414	if (unlikely(ret))
1415	return ret;
1416	return get_random_bytes_user(iter: &iter);
1417	}
1418
1419	static __poll_t random_poll(struct file *file, poll_table *wait)
1420	{
1421	poll_wait(filp: file, wait_address: &crng_init_wait, p: wait);
1422	return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM;
1423	}
1424
1425	static ssize_t write_pool_user(struct iov_iter *iter)
1426	{
1427	u8 block[BLAKE2S_BLOCK_SIZE];
1428	ssize_t ret = 0;
1429	size_t copied;
1430
1431	if (unlikely(!iov_iter_count(iter)))
1432	return 0;
1433
1434	for (;;) {
1435	copied = copy_from_iter(addr: block, bytes: sizeof(block), i: iter);
1436	ret += copied;
1437	mix_pool_bytes(buf: block, len: copied);
1438	if (!iov_iter_count(i: iter) || copied != sizeof(block))
1439	break;
1440
1441	BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
1442	if (ret % PAGE_SIZE == 0) {
1443	if (signal_pending(current))
1444	break;
1445	cond_resched();
1446	}
1447	}
1448
1449	memzero_explicit(s: block, count: sizeof(block));
1450	return ret ? ret : -EFAULT;
1451	}
1452
1453	static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
1454	{
1455	return write_pool_user(iter);
1456	}
1457
1458	static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
1459	{
1460	static int maxwarn = 10;
1461
1462	/*
1463	* Opportunistically attempt to initialize the RNG on platforms that
1464	* have fast cycle counters, but don't (for now) require it to succeed.
1465	*/
1466	if (!crng_ready())
1467	try_to_generate_entropy();
1468
1469	if (!crng_ready()) {
1470	if (!ratelimit_disable && maxwarn <= 0)
1471	ratelimit_state_inc_miss(rs: &urandom_warning);
1472	else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
1473	--maxwarn;
1474	pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
1475	current->comm, iov_iter_count(iter));
1476	}
1477	}
1478
1479	return get_random_bytes_user(iter);
1480	}
1481
1482	static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
1483	{
1484	int ret;
1485
1486	if (!crng_ready() &&
1487	((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) ||
1488	(kiocb->ki_filp->f_flags & O_NONBLOCK)))
1489	return -EAGAIN;
1490
1491	ret = wait_for_random_bytes();
1492	if (ret != 0)
1493	return ret;
1494	return get_random_bytes_user(iter);
1495	}
1496
1497	static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1498	{
1499	int __user *p = (int __user *)arg;
1500	int ent_count;
1501
1502	switch (cmd) {
1503	case RNDGETENTCNT:
1504	/* Inherently racy, no point locking. */
1505	if (put_user(input_pool.init_bits, p))
1506	return -EFAULT;
1507	return 0;
1508	case RNDADDTOENTCNT:
1509	if (!capable(CAP_SYS_ADMIN))
1510	return -EPERM;
1511	if (get_user(ent_count, p))
1512	return -EFAULT;
1513	if (ent_count < 0)
1514	return -EINVAL;
1515	credit_init_bits(ent_count);
1516	return 0;
1517	case RNDADDENTROPY: {
1518	struct iov_iter iter;
1519	ssize_t ret;
1520	int len;
1521
1522	if (!capable(CAP_SYS_ADMIN))
1523	return -EPERM;
1524	if (get_user(ent_count, p++))
1525	return -EFAULT;
1526	if (ent_count < 0)
1527	return -EINVAL;
1528	if (get_user(len, p++))
1529	return -EFAULT;
1530	ret = import_ubuf(ITER_SOURCE, buf: p, len, i: &iter);
1531	if (unlikely(ret))
1532	return ret;
1533	ret = write_pool_user(iter: &iter);
1534	if (unlikely(ret < 0))
1535	return ret;
1536	/* Since we're crediting, enforce that it was all written into the pool. */
1537	if (unlikely(ret != len))
1538	return -EFAULT;
1539	credit_init_bits(ent_count);
1540	return 0;
1541	}
1542	case RNDZAPENTCNT:
1543	case RNDCLEARPOOL:
1544	/* No longer has any effect. */
1545	if (!capable(CAP_SYS_ADMIN))
1546	return -EPERM;
1547	return 0;
1548	case RNDRESEEDCRNG:
1549	if (!capable(CAP_SYS_ADMIN))
1550	return -EPERM;
1551	if (!crng_ready())
1552	return -ENODATA;
1553	crng_reseed(NULL);
1554	return 0;
1555	default:
1556	return -EINVAL;
1557	}
1558	}
1559
1560	static int random_fasync(int fd, struct file *filp, int on)
1561	{
1562	return fasync_helper(fd, filp, on, &fasync);
1563	}
1564
1565	const struct file_operations random_fops = {
1566	.read_iter = random_read_iter,
1567	.write_iter = random_write_iter,
1568	.poll = random_poll,
1569	.unlocked_ioctl = random_ioctl,
1570	.compat_ioctl = compat_ptr_ioctl,
1571	.fasync = random_fasync,
1572	.llseek = noop_llseek,
1573	.splice_read = copy_splice_read,
1574	.splice_write = iter_file_splice_write,
1575	};
1576
1577	const struct file_operations urandom_fops = {
1578	.read_iter = urandom_read_iter,
1579	.write_iter = random_write_iter,
1580	.unlocked_ioctl = random_ioctl,
1581	.compat_ioctl = compat_ptr_ioctl,
1582	.fasync = random_fasync,
1583	.llseek = noop_llseek,
1584	.splice_read = copy_splice_read,
1585	.splice_write = iter_file_splice_write,
1586	};
1587
1588
1589	/********************************************************************
1590	*
1591	* Sysctl interface.
1592	*
1593	* These are partly unused legacy knobs with dummy values to not break
1594	* userspace and partly still useful things. They are usually accessible
1595	* in /proc/sys/kernel/random/ and are as follows:
1596	*
1597	* - boot_id - a UUID representing the current boot.
1598	*
1599	* - uuid - a random UUID, different each time the file is read.
1600	*
1601	* - poolsize - the number of bits of entropy that the input pool can
1602	* hold, tied to the POOL_BITS constant.
1603	*
1604	* - entropy_avail - the number of bits of entropy currently in the
1605	* input pool. Always <= poolsize.
1606	*
1607	* - write_wakeup_threshold - the amount of entropy in the input pool
1608	* below which write polls to /dev/random will unblock, requesting
1609	* more entropy, tied to the POOL_READY_BITS constant. It is writable
1610	* to avoid breaking old userspaces, but writing to it does not
1611	* change any behavior of the RNG.
1612	*
1613	* - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
1614	* It is writable to avoid breaking old userspaces, but writing
1615	* to it does not change any behavior of the RNG.
1616	*
1617	********************************************************************/
1618
1619	#ifdef CONFIG_SYSCTL
1620
1621	#include <linux/sysctl.h>
1622
1623	static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
1624	static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
1625	static int sysctl_poolsize = POOL_BITS;
1626	static u8 sysctl_bootid[UUID_SIZE];
1627
1628	/*
1629	* This function is used to return both the bootid UUID, and random
1630	* UUID. The difference is in whether table->data is NULL; if it is,
1631	* then a new UUID is generated and returned to the user.
1632	*/
1633	static int proc_do_uuid(const struct ctl_table *table, int write, void *buf,
1634	size_t *lenp, loff_t *ppos)
1635	{
1636	u8 tmp_uuid[UUID_SIZE], *uuid;
1637	char uuid_string[UUID_STRING_LEN + 1];
1638	struct ctl_table fake_table = {
1639	.data = uuid_string,
1640	.maxlen = UUID_STRING_LEN
1641	};
1642
1643	if (write)
1644	return -EPERM;
1645
1646	uuid = table->data;
1647	if (!uuid) {
1648	uuid = tmp_uuid;
1649	generate_random_uuid(uuid);
1650	} else {
1651	static DEFINE_SPINLOCK(bootid_spinlock);
1652
1653	spin_lock(lock: &bootid_spinlock);
1654	if (!uuid[8])
1655	generate_random_uuid(uuid);
1656	spin_unlock(lock: &bootid_spinlock);
1657	}
1658
1659	snprintf(buf: uuid_string, size: sizeof(uuid_string), fmt: "%pU", uuid);
1660	return proc_dostring(&fake_table, 0, buf, lenp, ppos);
1661	}
1662
1663	/* The same as proc_dointvec, but writes don't change anything. */
1664	static int proc_do_rointvec(const struct ctl_table *table, int write, void *buf,
1665	size_t *lenp, loff_t *ppos)
1666	{
1667	return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
1668	}
1669
1670	static const struct ctl_table random_table[] = {
1671	{
1672	.procname = "poolsize",
1673	.data = &sysctl_poolsize,
1674	.maxlen = sizeof(int),
1675	.mode = 0444,
1676	.proc_handler = proc_dointvec,
1677	},
1678	{
1679	.procname = "entropy_avail",
1680	.data = &input_pool.init_bits,
1681	.maxlen = sizeof(int),
1682	.mode = 0444,
1683	.proc_handler = proc_dointvec,
1684	},
1685	{
1686	.procname = "write_wakeup_threshold",
1687	.data = &sysctl_random_write_wakeup_bits,
1688	.maxlen = sizeof(int),
1689	.mode = 0644,
1690	.proc_handler = proc_do_rointvec,
1691	},
1692	{
1693	.procname = "urandom_min_reseed_secs",
1694	.data = &sysctl_random_min_urandom_seed,
1695	.maxlen = sizeof(int),
1696	.mode = 0644,
1697	.proc_handler = proc_do_rointvec,
1698	},
1699	{
1700	.procname = "boot_id",
1701	.data = &sysctl_bootid,
1702	.mode = 0444,
1703	.proc_handler = proc_do_uuid,
1704	},
1705	{
1706	.procname = "uuid",
1707	.mode = 0444,
1708	.proc_handler = proc_do_uuid,
1709	},
1710	};
1711
1712	/*
1713	* random_init() is called before sysctl_init(),
1714	* so we cannot call register_sysctl_init() in random_init()
1715	*/
1716	static int __init random_sysctls_init(void)
1717	{
1718	register_sysctl_init("kernel/random", random_table);
1719	return 0;
1720	}
1721	device_initcall(random_sysctls_init);
1722	#endif
1723