1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/list.h>
12#include <linux/mutex.h>
13#include <linux/slab.h>
14#include <linux/sort.h>
15
16#include <asm/fprobe.h>
17
18#include "trace.h"
19
20#define FPROBE_IP_HASH_BITS 8
21#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS)
22
23#define FPROBE_HASH_BITS 6
24#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS)
25
26#define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))
27
28/*
29 * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still
30 * exists. The key is the address of fprobe instance.
31 * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe
32 * instance related to the funciton address. The key is the ftrace IP
33 * address.
34 *
35 * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
36 * are set NULL and delete those from both hash tables (by hlist_del_rcu).
37 * After an RCU grace period, the fprobe_hlist itself will be released.
38 *
39 * fprobe_table and fprobe_ip_table can be accessed from either
40 * - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
41 * - RCU hlist traversal under disabling preempt
42 */
43static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
44static struct hlist_head fprobe_ip_table[FPROBE_IP_TABLE_SIZE];
45static DEFINE_MUTEX(fprobe_mutex);
46
47/*
48 * Find first fprobe in the hlist. It will be iterated twice in the entry
49 * probe, once for correcting the total required size, the second time is
50 * calling back the user handlers.
51 * Thus the hlist in the fprobe_table must be sorted and new probe needs to
52 * be added *before* the first fprobe.
53 */
54static struct fprobe_hlist_node *find_first_fprobe_node(unsigned long ip)
55{
56 struct fprobe_hlist_node *node;
57 struct hlist_head *head;
58
59 head = &fprobe_ip_table[hash_ptr(ptr: (void *)ip, FPROBE_IP_HASH_BITS)];
60 hlist_for_each_entry_rcu(node, head, hlist,
61 lockdep_is_held(&fprobe_mutex)) {
62 if (node->addr == ip)
63 return node;
64 }
65 return NULL;
66}
67NOKPROBE_SYMBOL(find_first_fprobe_node);
68
69/* Node insertion and deletion requires the fprobe_mutex */
70static void insert_fprobe_node(struct fprobe_hlist_node *node)
71{
72 unsigned long ip = node->addr;
73 struct fprobe_hlist_node *next;
74 struct hlist_head *head;
75
76 lockdep_assert_held(&fprobe_mutex);
77
78 next = find_first_fprobe_node(ip);
79 if (next) {
80 hlist_add_before_rcu(n: &node->hlist, next: &next->hlist);
81 return;
82 }
83 head = &fprobe_ip_table[hash_ptr(ptr: (void *)ip, FPROBE_IP_HASH_BITS)];
84 hlist_add_head_rcu(n: &node->hlist, h: head);
85}
86
87/* Return true if there are synonims */
88static bool delete_fprobe_node(struct fprobe_hlist_node *node)
89{
90 lockdep_assert_held(&fprobe_mutex);
91
92 /* Avoid double deleting */
93 if (READ_ONCE(node->fp) != NULL) {
94 WRITE_ONCE(node->fp, NULL);
95 hlist_del_rcu(n: &node->hlist);
96 }
97 return !!find_first_fprobe_node(ip: node->addr);
98}
99
100/* Check existence of the fprobe */
101static bool is_fprobe_still_exist(struct fprobe *fp)
102{
103 struct hlist_head *head;
104 struct fprobe_hlist *fph;
105
106 head = &fprobe_table[hash_ptr(ptr: fp, FPROBE_HASH_BITS)];
107 hlist_for_each_entry_rcu(fph, head, hlist,
108 lockdep_is_held(&fprobe_mutex)) {
109 if (fph->fp == fp)
110 return true;
111 }
112 return false;
113}
114NOKPROBE_SYMBOL(is_fprobe_still_exist);
115
116static int add_fprobe_hash(struct fprobe *fp)
117{
118 struct fprobe_hlist *fph = fp->hlist_array;
119 struct hlist_head *head;
120
121 lockdep_assert_held(&fprobe_mutex);
122
123 if (WARN_ON_ONCE(!fph))
124 return -EINVAL;
125
126 if (is_fprobe_still_exist(fp))
127 return -EEXIST;
128
129 head = &fprobe_table[hash_ptr(ptr: fp, FPROBE_HASH_BITS)];
130 hlist_add_head_rcu(n: &fp->hlist_array->hlist, h: head);
131 return 0;
132}
133
134static int del_fprobe_hash(struct fprobe *fp)
135{
136 struct fprobe_hlist *fph = fp->hlist_array;
137
138 lockdep_assert_held(&fprobe_mutex);
139
140 if (WARN_ON_ONCE(!fph))
141 return -EINVAL;
142
143 if (!is_fprobe_still_exist(fp))
144 return -ENOENT;
145
146 fph->fp = NULL;
147 hlist_del_rcu(n: &fph->hlist);
148 return 0;
149}
150
151#ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER
152
153/* The arch should encode fprobe_header info into one unsigned long */
154#define FPROBE_HEADER_SIZE_IN_LONG 1
155
156static inline bool write_fprobe_header(unsigned long *stack,
157 struct fprobe *fp, unsigned int size_words)
158{
159 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD ||
160 !arch_fprobe_header_encodable(fp)))
161 return false;
162
163 *stack = arch_encode_fprobe_header(fp, size_words);
164 return true;
165}
166
167static inline void read_fprobe_header(unsigned long *stack,
168 struct fprobe **fp, unsigned int *size_words)
169{
170 *fp = arch_decode_fprobe_header_fp(*stack);
171 *size_words = arch_decode_fprobe_header_size(*stack);
172}
173
174#else
175
176/* Generic fprobe_header */
177struct __fprobe_header {
178 struct fprobe *fp;
179 unsigned long size_words;
180} __packed;
181
182#define FPROBE_HEADER_SIZE_IN_LONG SIZE_IN_LONG(sizeof(struct __fprobe_header))
183
184static inline bool write_fprobe_header(unsigned long *stack,
185 struct fprobe *fp, unsigned int size_words)
186{
187 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
188
189 if (WARN_ON_ONCE(size_words > MAX_FPROBE_DATA_SIZE_WORD))
190 return false;
191
192 fph->fp = fp;
193 fph->size_words = size_words;
194 return true;
195}
196
197static inline void read_fprobe_header(unsigned long *stack,
198 struct fprobe **fp, unsigned int *size_words)
199{
200 struct __fprobe_header *fph = (struct __fprobe_header *)stack;
201
202 *fp = fph->fp;
203 *size_words = fph->size_words;
204}
205
206#endif
207
208/*
209 * fprobe shadow stack management:
210 * Since fprobe shares a single fgraph_ops, it needs to share the stack entry
211 * among the probes on the same function exit. Note that a new probe can be
212 * registered before a target function is returning, we can not use the hash
213 * table to find the corresponding probes. Thus the probe address is stored on
214 * the shadow stack with its entry data size.
215 *
216 */
217static inline int __fprobe_handler(unsigned long ip, unsigned long parent_ip,
218 struct fprobe *fp, struct ftrace_regs *fregs,
219 void *data)
220{
221 if (!fp->entry_handler)
222 return 0;
223
224 return fp->entry_handler(fp, ip, parent_ip, fregs, data);
225}
226
227static inline int __fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
228 struct fprobe *fp, struct ftrace_regs *fregs,
229 void *data)
230{
231 int ret;
232 /*
233 * This user handler is shared with other kprobes and is not expected to be
234 * called recursively. So if any other kprobe handler is running, this will
235 * exit as kprobe does. See the section 'Share the callbacks with kprobes'
236 * in Documentation/trace/fprobe.rst for more information.
237 */
238 if (unlikely(kprobe_running())) {
239 fp->nmissed++;
240 return 0;
241 }
242
243 kprobe_busy_begin();
244 ret = __fprobe_handler(ip, parent_ip, fp, fregs, data);
245 kprobe_busy_end();
246 return ret;
247}
248
249static int fprobe_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
250 struct ftrace_regs *fregs)
251{
252 struct fprobe_hlist_node *node, *first;
253 unsigned long *fgraph_data = NULL;
254 unsigned long func = trace->func;
255 unsigned long ret_ip;
256 int reserved_words;
257 struct fprobe *fp;
258 int used, ret;
259
260 if (WARN_ON_ONCE(!fregs))
261 return 0;
262
263 first = node = find_first_fprobe_node(ip: func);
264 if (unlikely(!first))
265 return 0;
266
267 reserved_words = 0;
268 hlist_for_each_entry_from_rcu(node, hlist) {
269 if (node->addr != func)
270 break;
271 fp = READ_ONCE(node->fp);
272 if (!fp || !fp->exit_handler)
273 continue;
274 /*
275 * Since fprobe can be enabled until the next loop, we ignore the
276 * fprobe's disabled flag in this loop.
277 */
278 reserved_words +=
279 FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);
280 }
281 node = first;
282 if (reserved_words) {
283 fgraph_data = fgraph_reserve_data(idx: gops->idx, size_bytes: reserved_words * sizeof(long));
284 if (unlikely(!fgraph_data)) {
285 hlist_for_each_entry_from_rcu(node, hlist) {
286 if (node->addr != func)
287 break;
288 fp = READ_ONCE(node->fp);
289 if (fp && !fprobe_disabled(fp))
290 fp->nmissed++;
291 }
292 return 0;
293 }
294 }
295
296 /*
297 * TODO: recursion detection has been done in the fgraph. Thus we need
298 * to add a callback to increment missed counter.
299 */
300 ret_ip = ftrace_regs_get_return_address(fregs);
301 used = 0;
302 hlist_for_each_entry_from_rcu(node, hlist) {
303 int data_size;
304 void *data;
305
306 if (node->addr != func)
307 break;
308 fp = READ_ONCE(node->fp);
309 if (!fp || fprobe_disabled(fp))
310 continue;
311
312 data_size = fp->entry_data_size;
313 if (data_size && fp->exit_handler)
314 data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG;
315 else
316 data = NULL;
317
318 if (fprobe_shared_with_kprobes(fp))
319 ret = __fprobe_kprobe_handler(ip: func, parent_ip: ret_ip, fp, fregs, data);
320 else
321 ret = __fprobe_handler(ip: func, parent_ip: ret_ip, fp, fregs, data);
322
323 /* If entry_handler returns !0, nmissed is not counted but skips exit_handler. */
324 if (!ret && fp->exit_handler) {
325 int size_words = SIZE_IN_LONG(data_size);
326
327 if (write_fprobe_header(stack: &fgraph_data[used], fp, size_words))
328 used += FPROBE_HEADER_SIZE_IN_LONG + size_words;
329 }
330 }
331 if (used < reserved_words)
332 memset(fgraph_data + used, 0, reserved_words - used);
333
334 /* If any exit_handler is set, data must be used. */
335 return used != 0;
336}
337NOKPROBE_SYMBOL(fprobe_entry);
338
339static void fprobe_return(struct ftrace_graph_ret *trace,
340 struct fgraph_ops *gops,
341 struct ftrace_regs *fregs)
342{
343 unsigned long *fgraph_data = NULL;
344 unsigned long ret_ip;
345 struct fprobe *fp;
346 int size, curr;
347 int size_words;
348
349 fgraph_data = (unsigned long *)fgraph_retrieve_data(idx: gops->idx, size_bytes: &size);
350 if (WARN_ON_ONCE(!fgraph_data))
351 return;
352 size_words = SIZE_IN_LONG(size);
353 ret_ip = ftrace_regs_get_instruction_pointer(fregs);
354
355 preempt_disable();
356
357 curr = 0;
358 while (size_words > curr) {
359 read_fprobe_header(stack: &fgraph_data[curr], fp: &fp, size_words: &size);
360 if (!fp)
361 break;
362 curr += FPROBE_HEADER_SIZE_IN_LONG;
363 if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) {
364 if (WARN_ON_ONCE(curr + size > size_words))
365 break;
366 fp->exit_handler(fp, trace->func, ret_ip, fregs,
367 size ? fgraph_data + curr : NULL);
368 }
369 curr += size;
370 }
371 preempt_enable();
372}
373NOKPROBE_SYMBOL(fprobe_return);
374
375static struct fgraph_ops fprobe_graph_ops = {
376 .entryfunc = fprobe_entry,
377 .retfunc = fprobe_return,
378};
379static int fprobe_graph_active;
380
381/* Add @addrs to the ftrace filter and register fgraph if needed. */
382static int fprobe_graph_add_ips(unsigned long *addrs, int num)
383{
384 int ret;
385
386 lockdep_assert_held(&fprobe_mutex);
387
388 ret = ftrace_set_filter_ips(ops: &fprobe_graph_ops.ops, ips: addrs, cnt: num, remove: 0, reset: 0);
389 if (ret)
390 return ret;
391
392 if (!fprobe_graph_active) {
393 ret = register_ftrace_graph(ops: &fprobe_graph_ops);
394 if (WARN_ON_ONCE(ret)) {
395 ftrace_free_filter(ops: &fprobe_graph_ops.ops);
396 return ret;
397 }
398 }
399 fprobe_graph_active++;
400 return 0;
401}
402
403/* Remove @addrs from the ftrace filter and unregister fgraph if possible. */
404static void fprobe_graph_remove_ips(unsigned long *addrs, int num)
405{
406 lockdep_assert_held(&fprobe_mutex);
407
408 fprobe_graph_active--;
409 /* Q: should we unregister it ? */
410 if (!fprobe_graph_active)
411 unregister_ftrace_graph(ops: &fprobe_graph_ops);
412
413 if (num)
414 ftrace_set_filter_ips(ops: &fprobe_graph_ops.ops, ips: addrs, cnt: num, remove: 1, reset: 0);
415}
416
417#ifdef CONFIG_MODULES
418
419#define FPROBE_IPS_BATCH_INIT 8
420/* instruction pointer address list */
421struct fprobe_addr_list {
422 int index;
423 int size;
424 unsigned long *addrs;
425};
426
427static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned long addr)
428{
429 unsigned long *addrs;
430
431 if (alist->index >= alist->size)
432 return -ENOMEM;
433
434 alist->addrs[alist->index++] = addr;
435 if (alist->index < alist->size)
436 return 0;
437
438 /* Expand the address list */
439 addrs = kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL);
440 if (!addrs)
441 return -ENOMEM;
442
443 memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs));
444 alist->size *= 2;
445 kfree(objp: alist->addrs);
446 alist->addrs = addrs;
447
448 return 0;
449}
450
451static void fprobe_remove_node_in_module(struct module *mod, struct hlist_head *head,
452 struct fprobe_addr_list *alist)
453{
454 struct fprobe_hlist_node *node;
455 int ret = 0;
456
457 hlist_for_each_entry_rcu(node, head, hlist,
458 lockdep_is_held(&fprobe_mutex)) {
459 if (!within_module(addr: node->addr, mod))
460 continue;
461 if (delete_fprobe_node(node))
462 continue;
463 /*
464 * If failed to update alist, just continue to update hlist.
465 * Therefore, at list user handler will not hit anymore.
466 */
467 if (!ret)
468 ret = fprobe_addr_list_add(alist, addr: node->addr);
469 }
470}
471
472/* Handle module unloading to manage fprobe_ip_table. */
473static int fprobe_module_callback(struct notifier_block *nb,
474 unsigned long val, void *data)
475{
476 struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT};
477 struct module *mod = data;
478 int i;
479
480 if (val != MODULE_STATE_GOING)
481 return NOTIFY_DONE;
482
483 alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL);
484 /* If failed to alloc memory, we can not remove ips from hash. */
485 if (!alist.addrs)
486 return NOTIFY_DONE;
487
488 mutex_lock(&fprobe_mutex);
489 for (i = 0; i < FPROBE_IP_TABLE_SIZE; i++)
490 fprobe_remove_node_in_module(mod, head: &fprobe_ip_table[i], alist: &alist);
491
492 if (alist.index < alist.size && alist.index > 0)
493 ftrace_set_filter_ips(ops: &fprobe_graph_ops.ops,
494 ips: alist.addrs, cnt: alist.index, remove: 1, reset: 0);
495 mutex_unlock(lock: &fprobe_mutex);
496
497 kfree(objp: alist.addrs);
498
499 return NOTIFY_DONE;
500}
501
502static struct notifier_block fprobe_module_nb = {
503 .notifier_call = fprobe_module_callback,
504 .priority = 0,
505};
506
507static int __init init_fprobe_module(void)
508{
509 return register_module_notifier(nb: &fprobe_module_nb);
510}
511early_initcall(init_fprobe_module);
512#endif
513
514static int symbols_cmp(const void *a, const void *b)
515{
516 const char **str_a = (const char **) a;
517 const char **str_b = (const char **) b;
518
519 return strcmp(*str_a, *str_b);
520}
521
522/* Convert ftrace location address from symbols */
523static unsigned long *get_ftrace_locations(const char **syms, int num)
524{
525 unsigned long *addrs;
526
527 /* Convert symbols to symbol address */
528 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
529 if (!addrs)
530 return ERR_PTR(error: -ENOMEM);
531
532 /* ftrace_lookup_symbols expects sorted symbols */
533 sort(base: syms, num, size: sizeof(*syms), cmp_func: symbols_cmp, NULL);
534
535 if (!ftrace_lookup_symbols(sorted_syms: syms, cnt: num, addrs))
536 return addrs;
537
538 kfree(objp: addrs);
539 return ERR_PTR(error: -ENOENT);
540}
541
542struct filter_match_data {
543 const char *filter;
544 const char *notfilter;
545 size_t index;
546 size_t size;
547 unsigned long *addrs;
548 struct module **mods;
549};
550
551static int filter_match_callback(void *data, const char *name, unsigned long addr)
552{
553 struct filter_match_data *match = data;
554
555 if (!glob_match(pat: match->filter, str: name) ||
556 (match->notfilter && glob_match(pat: match->notfilter, str: name)))
557 return 0;
558
559 if (!ftrace_location(ip: addr))
560 return 0;
561
562 if (match->addrs) {
563 struct module *mod = __module_text_address(addr);
564
565 if (mod && !try_module_get(module: mod))
566 return 0;
567
568 match->mods[match->index] = mod;
569 match->addrs[match->index] = addr;
570 }
571 match->index++;
572 return match->index == match->size;
573}
574
575/*
576 * Make IP list from the filter/no-filter glob patterns.
577 * Return the number of matched symbols, or errno.
578 * If @addrs == NULL, this just counts the number of matched symbols. If @addrs
579 * is passed with an array, we need to pass the an @mods array of the same size
580 * to increment the module refcount for each symbol.
581 * This means we also need to call `module_put` for each element of @mods after
582 * using the @addrs.
583 */
584static int get_ips_from_filter(const char *filter, const char *notfilter,
585 unsigned long *addrs, struct module **mods,
586 size_t size)
587{
588 struct filter_match_data match = { .filter = filter, .notfilter = notfilter,
589 .index = 0, .size = size, .addrs = addrs, .mods = mods};
590 int ret;
591
592 if (addrs && !mods)
593 return -EINVAL;
594
595 ret = kallsyms_on_each_symbol(fn: filter_match_callback, data: &match);
596 if (ret < 0)
597 return ret;
598 if (IS_ENABLED(CONFIG_MODULES)) {
599 ret = module_kallsyms_on_each_symbol(NULL, fn: filter_match_callback, data: &match);
600 if (ret < 0)
601 return ret;
602 }
603
604 return match.index ?: -ENOENT;
605}
606
607static void fprobe_fail_cleanup(struct fprobe *fp)
608{
609 kfree(objp: fp->hlist_array);
610 fp->hlist_array = NULL;
611}
612
613/* Initialize the fprobe data structure. */
614static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)
615{
616 struct fprobe_hlist *hlist_array;
617 unsigned long addr;
618 int size, i;
619
620 if (!fp || !addrs || num <= 0)
621 return -EINVAL;
622
623 size = ALIGN(fp->entry_data_size, sizeof(long));
624 if (size > MAX_FPROBE_DATA_SIZE)
625 return -E2BIG;
626 fp->entry_data_size = size;
627
628 hlist_array = kzalloc(struct_size(hlist_array, array, num), GFP_KERNEL);
629 if (!hlist_array)
630 return -ENOMEM;
631
632 fp->nmissed = 0;
633
634 hlist_array->size = num;
635 fp->hlist_array = hlist_array;
636 hlist_array->fp = fp;
637 for (i = 0; i < num; i++) {
638 hlist_array->array[i].fp = fp;
639 addr = ftrace_location(ip: addrs[i]);
640 if (!addr) {
641 fprobe_fail_cleanup(fp);
642 return -ENOENT;
643 }
644 hlist_array->array[i].addr = addr;
645 }
646 return 0;
647}
648
649#define FPROBE_IPS_MAX INT_MAX
650
651/**
652 * register_fprobe() - Register fprobe to ftrace by pattern.
653 * @fp: A fprobe data structure to be registered.
654 * @filter: A wildcard pattern of probed symbols.
655 * @notfilter: A wildcard pattern of NOT probed symbols.
656 *
657 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
658 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
659 *
660 * Return 0 if @fp is registered successfully, -errno if not.
661 */
662int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
663{
664 unsigned long *addrs __free(kfree) = NULL;
665 struct module **mods __free(kfree) = NULL;
666 int ret, num;
667
668 if (!fp || !filter)
669 return -EINVAL;
670
671 num = get_ips_from_filter(filter, notfilter, NULL, NULL, FPROBE_IPS_MAX);
672 if (num < 0)
673 return num;
674
675 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
676 if (!addrs)
677 return -ENOMEM;
678
679 mods = kcalloc(num, sizeof(*mods), GFP_KERNEL);
680 if (!mods)
681 return -ENOMEM;
682
683 ret = get_ips_from_filter(filter, notfilter, addrs, mods, size: num);
684 if (ret < 0)
685 return ret;
686
687 ret = register_fprobe_ips(fp, addrs, num: ret);
688
689 for (int i = 0; i < num; i++) {
690 if (mods[i])
691 module_put(module: mods[i]);
692 }
693 return ret;
694}
695EXPORT_SYMBOL_GPL(register_fprobe);
696
697/**
698 * register_fprobe_ips() - Register fprobe to ftrace by address.
699 * @fp: A fprobe data structure to be registered.
700 * @addrs: An array of target function address.
701 * @num: The number of entries of @addrs.
702 *
703 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
704 * The @addrs must be the addresses of ftrace location address, which may be
705 * the symbol address + arch-dependent offset.
706 * If you unsure what this mean, please use other registration functions.
707 *
708 * Return 0 if @fp is registered successfully, -errno if not.
709 */
710int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
711{
712 struct fprobe_hlist *hlist_array;
713 int ret, i;
714
715 ret = fprobe_init(fp, addrs, num);
716 if (ret)
717 return ret;
718
719 mutex_lock(&fprobe_mutex);
720
721 hlist_array = fp->hlist_array;
722 ret = fprobe_graph_add_ips(addrs, num);
723 if (!ret) {
724 add_fprobe_hash(fp);
725 for (i = 0; i < hlist_array->size; i++)
726 insert_fprobe_node(node: &hlist_array->array[i]);
727 }
728 mutex_unlock(lock: &fprobe_mutex);
729
730 if (ret)
731 fprobe_fail_cleanup(fp);
732
733 return ret;
734}
735EXPORT_SYMBOL_GPL(register_fprobe_ips);
736
737/**
738 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
739 * @fp: A fprobe data structure to be registered.
740 * @syms: An array of target symbols.
741 * @num: The number of entries of @syms.
742 *
743 * Register @fp to the symbols given by @syms array. This will be useful if
744 * you are sure the symbols exist in the kernel.
745 *
746 * Return 0 if @fp is registered successfully, -errno if not.
747 */
748int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
749{
750 unsigned long *addrs;
751 int ret;
752
753 if (!fp || !syms || num <= 0)
754 return -EINVAL;
755
756 addrs = get_ftrace_locations(syms, num);
757 if (IS_ERR(ptr: addrs))
758 return PTR_ERR(ptr: addrs);
759
760 ret = register_fprobe_ips(fp, addrs, num);
761
762 kfree(objp: addrs);
763
764 return ret;
765}
766EXPORT_SYMBOL_GPL(register_fprobe_syms);
767
768bool fprobe_is_registered(struct fprobe *fp)
769{
770 if (!fp || !fp->hlist_array)
771 return false;
772 return true;
773}
774
775/**
776 * unregister_fprobe() - Unregister fprobe.
777 * @fp: A fprobe data structure to be unregistered.
778 *
779 * Unregister fprobe (and remove ftrace hooks from the function entries).
780 *
781 * Return 0 if @fp is unregistered successfully, -errno if not.
782 */
783int unregister_fprobe(struct fprobe *fp)
784{
785 struct fprobe_hlist *hlist_array;
786 unsigned long *addrs = NULL;
787 int ret = 0, i, count;
788
789 mutex_lock(&fprobe_mutex);
790 if (!fp || !is_fprobe_still_exist(fp)) {
791 ret = -EINVAL;
792 goto out;
793 }
794
795 hlist_array = fp->hlist_array;
796 addrs = kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL);
797 if (!addrs) {
798 ret = -ENOMEM; /* TODO: Fallback to one-by-one loop */
799 goto out;
800 }
801
802 /* Remove non-synonim ips from table and hash */
803 count = 0;
804 for (i = 0; i < hlist_array->size; i++) {
805 if (!delete_fprobe_node(node: &hlist_array->array[i]))
806 addrs[count++] = hlist_array->array[i].addr;
807 }
808 del_fprobe_hash(fp);
809
810 fprobe_graph_remove_ips(addrs, num: count);
811
812 kfree_rcu(hlist_array, rcu);
813 fp->hlist_array = NULL;
814
815out:
816 mutex_unlock(lock: &fprobe_mutex);
817
818 kfree(objp: addrs);
819 return ret;
820}
821EXPORT_SYMBOL_GPL(unregister_fprobe);
822

source code of linux/kernel/trace/fprobe.c

Morty Proxy This is a proxified and sanitized view of the page, visit original site.