Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067// SPDX-License-Identifier: GPL-2.0#include <linux/fsnotify.h>
#include <asm/setup.h> /* COMMAND_LINE_SIZE */
#include "trace.h"
/* Used if snapshot allocated at boot */static bool allocate_snapshot;static bool snapshot_at_boot;
static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;static int boot_snapshot_index;
static int __init boot_alloc_snapshot(char *str){ char *slot = boot_snapshot_info + boot_snapshot_index; int left = sizeof(boot_snapshot_info) - boot_snapshot_index; int ret;
if (str[0] == '=') { str++; if (strlen(str) >= left) return -1;
ret = snprintf(slot, left, "%s\t", str); boot_snapshot_index += ret; } else { allocate_snapshot = true; /* We also need the main ring buffer expanded */ trace_set_ring_buffer_expanded(NULL); } return 1;}__setup("alloc_snapshot", boot_alloc_snapshot);
static int __init boot_snapshot(char *str){ snapshot_at_boot = true; boot_alloc_snapshot(str); return 1;}__setup("ftrace_boot_snapshot", boot_snapshot);static void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data){ unsigned long flags;
if (in_nmi()) { trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); trace_array_puts(tr, "*** snapshot is being ignored ***\n"); return; }
if (!tr->allocated_snapshot) { trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n"); trace_array_puts(tr, "*** stopping trace here! ***\n"); tracer_tracing_off(tr); return; }
if (tr->mapped) { trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n"); trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); return; }
/* Note, snapshot can not be used when the tracer uses it */ if (tracer_uses_snapshot(tr->current_trace)) { trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n"); trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); return; }
local_irq_save(flags); update_max_tr(tr, current, smp_processor_id(), cond_data); local_irq_restore(flags);}
void tracing_snapshot_instance(struct trace_array *tr){ tracing_snapshot_instance_cond(tr, NULL);}
/** * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. * @tr: The tracing instance to snapshot * @cond_data: The data to be tested conditionally, and possibly saved * * This is the same as tracing_snapshot() except that the snapshot is * conditional - the snapshot will only happen if the * cond_snapshot.update() implementation receiving the cond_data * returns true, which means that the trace array's cond_snapshot * update() operation used the cond_data to determine whether the * snapshot should be taken, and if it was, presumably saved it along * with the snapshot. */void tracing_snapshot_cond(struct trace_array *tr, void *cond_data){ tracing_snapshot_instance_cond(tr, cond_data);}EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
/** * tracing_cond_snapshot_data - get the user data associated with a snapshot * @tr: The tracing instance * * When the user enables a conditional snapshot using * tracing_snapshot_cond_enable(), the user-defined cond_data is saved * with the snapshot. This accessor is used to retrieve it. * * Should not be called from cond_snapshot.update(), since it takes * the tr->max_lock lock, which the code calling * cond_snapshot.update() has already done. * * Returns the cond_data associated with the trace array's snapshot. */void *tracing_cond_snapshot_data(struct trace_array *tr){ void *cond_data = NULL;
local_irq_disable(); arch_spin_lock(&tr->max_lock);
if (tr->cond_snapshot) cond_data = tr->cond_snapshot->cond_data;
arch_spin_unlock(&tr->max_lock); local_irq_enable();
return cond_data;}EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
/* resize @tr's buffer to the size of @size_tr's entries */int resize_buffer_duplicate_size(struct array_buffer *trace_buf, struct array_buffer *size_buf, int cpu_id){ int cpu, ret = 0;
if (cpu_id == RING_BUFFER_ALL_CPUS) { for_each_tracing_cpu(cpu) { ret = ring_buffer_resize(trace_buf->buffer, per_cpu_ptr(size_buf->data, cpu)->entries, cpu); if (ret < 0) break; per_cpu_ptr(trace_buf->data, cpu)->entries = per_cpu_ptr(size_buf->data, cpu)->entries; } } else { ret = ring_buffer_resize(trace_buf->buffer, per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); if (ret == 0) per_cpu_ptr(trace_buf->data, cpu_id)->entries = per_cpu_ptr(size_buf->data, cpu_id)->entries; }
return ret;}
int tracing_alloc_snapshot_instance(struct trace_array *tr){ int order; int ret;
if (!tr->allocated_snapshot) {
/* Make the snapshot buffer have the same order as main buffer */ order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); ret = ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, order); if (ret < 0) return ret;
/* allocate spare buffer */ ret = resize_buffer_duplicate_size(&tr->snapshot_buffer, &tr->array_buffer, RING_BUFFER_ALL_CPUS); if (ret < 0) return ret;
tr->allocated_snapshot = true; }
return 0;}
void free_snapshot(struct trace_array *tr){ /* * We don't free the ring buffer. instead, resize it because * The max_tr ring buffer has some state (e.g. ring->clock) and * we want preserve it. */ ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, 0); ring_buffer_resize(tr->snapshot_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); trace_set_buffer_entries(&tr->snapshot_buffer, 1); tracing_reset_online_cpus(&tr->snapshot_buffer); tr->allocated_snapshot = false;}
int tracing_arm_snapshot_locked(struct trace_array *tr){ int ret;
lockdep_assert_held(&trace_types_lock);
spin_lock(&tr->snapshot_trigger_lock); if (tr->snapshot == UINT_MAX || tr->mapped) { spin_unlock(&tr->snapshot_trigger_lock); return -EBUSY; }
tr->snapshot++; spin_unlock(&tr->snapshot_trigger_lock);
ret = tracing_alloc_snapshot_instance(tr); if (ret) { spin_lock(&tr->snapshot_trigger_lock); tr->snapshot--; spin_unlock(&tr->snapshot_trigger_lock); }
return ret;}
int tracing_arm_snapshot(struct trace_array *tr){ guard(mutex)(&trace_types_lock); return tracing_arm_snapshot_locked(tr);}
void tracing_disarm_snapshot(struct trace_array *tr){ spin_lock(&tr->snapshot_trigger_lock); if (!WARN_ON(!tr->snapshot)) tr->snapshot--; spin_unlock(&tr->snapshot_trigger_lock);}
/** * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. * * This is similar to tracing_snapshot(), but it will allocate the * snapshot buffer if it isn't already allocated. Use this only * where it is safe to sleep, as the allocation may sleep. * * This causes a swap between the snapshot buffer and the current live * tracing buffer. You can use this to take snapshots of the live * trace when some condition is triggered, but continue to trace. */void tracing_snapshot_alloc(void){ int ret;
ret = tracing_alloc_snapshot(); if (ret < 0) return;
tracing_snapshot();}EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
/** * tracing_snapshot_cond_enable - enable conditional snapshot for an instance * @tr: The tracing instance * @cond_data: User data to associate with the snapshot * @update: Implementation of the cond_snapshot update function * * Check whether the conditional snapshot for the given instance has * already been enabled, or if the current tracer is already using a * snapshot; if so, return -EBUSY, else create a cond_snapshot and * save the cond_data and update function inside. * * Returns 0 if successful, error otherwise. */int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update){ struct cond_snapshot *cond_snapshot __free(kfree) = kzalloc_obj(*cond_snapshot); int ret;
if (!cond_snapshot) return -ENOMEM;
cond_snapshot->cond_data = cond_data; cond_snapshot->update = update;
guard(mutex)(&trace_types_lock);
if (tracer_uses_snapshot(tr->current_trace)) return -EBUSY;
/* * The cond_snapshot can only change to NULL without the * trace_types_lock. We don't care if we race with it going * to NULL, but we want to make sure that it's not set to * something other than NULL when we get here, which we can * do safely with only holding the trace_types_lock and not * having to take the max_lock. */ if (tr->cond_snapshot) return -EBUSY;
ret = tracing_arm_snapshot_locked(tr); if (ret) return ret;
local_irq_disable(); arch_spin_lock(&tr->max_lock); tr->cond_snapshot = no_free_ptr(cond_snapshot); arch_spin_unlock(&tr->max_lock); local_irq_enable();
return 0;}EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
/** * tracing_snapshot_cond_disable - disable conditional snapshot for an instance * @tr: The tracing instance * * Check whether the conditional snapshot for the given instance is * enabled; if so, free the cond_snapshot associated with it, * otherwise return -EINVAL. * * Returns 0 if successful, error otherwise. */int tracing_snapshot_cond_disable(struct trace_array *tr){ int ret = 0;
local_irq_disable(); arch_spin_lock(&tr->max_lock);
if (!tr->cond_snapshot) ret = -EINVAL; else { kfree(tr->cond_snapshot); tr->cond_snapshot = NULL; }
arch_spin_unlock(&tr->max_lock); local_irq_enable();
tracing_disarm_snapshot(tr);
return ret;}EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
#ifdef CONFIG_TRACER_MAX_TRACE#ifdef LATENCY_FS_NOTIFYstatic struct workqueue_struct *fsnotify_wq;
static void latency_fsnotify_workfn(struct work_struct *work){ struct trace_array *tr = container_of(work, struct trace_array, fsnotify_work); fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);}
static void latency_fsnotify_workfn_irq(struct irq_work *iwork){ struct trace_array *tr = container_of(iwork, struct trace_array, fsnotify_irqwork); queue_work(fsnotify_wq, &tr->fsnotify_work);}
__init static int latency_fsnotify_init(void){ fsnotify_wq = alloc_workqueue("tr_max_lat_wq", WQ_UNBOUND | WQ_HIGHPRI, 0); if (!fsnotify_wq) { pr_err("Unable to allocate tr_max_lat_wq\n"); return -ENOMEM; } return 0;}
late_initcall_sync(latency_fsnotify_init);
void latency_fsnotify(struct trace_array *tr){ if (!fsnotify_wq) return; /* * We cannot call queue_work(&tr->fsnotify_work) from here because it's * possible that we are called from __schedule() or do_idle(), which * could cause a deadlock. */ irq_work_queue(&tr->fsnotify_irqwork);}#endif /* LATENCY_FS_NOTIFY */
static const struct file_operations tracing_max_lat_fops;
void trace_create_maxlat_file(struct trace_array *tr, struct dentry *d_tracer){#ifdef LATENCY_FS_NOTIFY INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);#endif tr->d_max_latency = trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, d_tracer, tr, &tracing_max_lat_fops);}
/* * Copy the new maximum trace into the separate maximum-trace * structure. (this way the maximum trace is permanently saved, * for later retrieval via /sys/kernel/tracing/tracing_max_latency) */static void__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu){ struct array_buffer *trace_buf = &tr->array_buffer; struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); struct array_buffer *max_buf = &tr->snapshot_buffer; struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
max_buf->cpu = cpu; max_buf->time_start = data->preempt_timestamp;
max_data->saved_latency = tr->max_latency; max_data->critical_start = data->critical_start; max_data->critical_end = data->critical_end;
strscpy(max_data->comm, tsk->comm); max_data->pid = tsk->pid; /* * If tsk == current, then use current_uid(), as that does not use * RCU. The irq tracer can be called out of RCU scope. */ if (tsk == current) max_data->uid = current_uid(); else max_data->uid = task_uid(tsk);
max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; max_data->policy = tsk->policy; max_data->rt_priority = tsk->rt_priority;
/* record this tasks comm */ tracing_record_cmdline(tsk); latency_fsnotify(tr);}#elsestatic inline void __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) { }#endif /* CONFIG_TRACER_MAX_TRACE */
/** * update_max_tr - snapshot all trace buffers from global_trace to max_tr * @tr: tracer * @tsk: the task with the latency * @cpu: The cpu that initiated the trace. * @cond_data: User data associated with a conditional snapshot * * Flip the buffers between the @tr and the max_tr and record information * about which task was the cause of this latency. */voidupdate_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, void *cond_data){ if (tr->stop_count) return;
WARN_ON_ONCE(!irqs_disabled());
if (!tr->allocated_snapshot) { /* Only the nop tracer should hit this when disabling */ WARN_ON_ONCE(tr->current_trace != &nop_trace); return; }
arch_spin_lock(&tr->max_lock);
/* Inherit the recordable setting from array_buffer */ if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) ring_buffer_record_on(tr->snapshot_buffer.buffer); else ring_buffer_record_off(tr->snapshot_buffer.buffer);
if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { arch_spin_unlock(&tr->max_lock); return; }
swap(tr->array_buffer.buffer, tr->snapshot_buffer.buffer);
__update_max_tr(tr, tsk, cpu);
arch_spin_unlock(&tr->max_lock);
/* Any waiters on the old snapshot buffer need to wake up */ ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS);}
/** * update_max_tr_single - only copy one trace over, and reset the rest * @tr: tracer * @tsk: task with the latency * @cpu: the cpu of the buffer to copy. * * Flip the trace of a single CPU buffer between the @tr and the max_tr. */voidupdate_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu){ int ret;
if (tr->stop_count) return;
WARN_ON_ONCE(!irqs_disabled()); if (!tr->allocated_snapshot) { /* Only the nop tracer should hit this when disabling */ WARN_ON_ONCE(tr->current_trace != &nop_trace); return; }
arch_spin_lock(&tr->max_lock);
ret = ring_buffer_swap_cpu(tr->snapshot_buffer.buffer, tr->array_buffer.buffer, cpu);
if (ret == -EBUSY) { /* * We failed to swap the buffer due to a commit taking * place on this CPU. We fail to record, but we reset * the max trace buffer (no one writes directly to it) * and flag that it failed. * Another reason is resize is in progress. */ trace_array_printk_buf(tr->snapshot_buffer.buffer, _THIS_IP_, "Failed to swap buffers due to commit or resize in progress\n"); }
WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
__update_max_tr(tr, tsk, cpu); arch_spin_unlock(&tr->max_lock);}
static void show_snapshot_main_help(struct seq_file *m){ seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" "# Takes a snapshot of the main buffer.\n" "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" "# (Doesn't have to be '2' works with any number that\n" "# is not a '0' or '1')\n");}
static void show_snapshot_percpu_help(struct seq_file *m){ seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" "# Takes a snapshot of the main buffer for this cpu.\n");#else seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" "# Must use main snapshot file to allocate.\n");#endif seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" "# (Doesn't have to be '2' works with any number that\n" "# is not a '0' or '1')\n");}
void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter){ if (iter->tr->allocated_snapshot) seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); else seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
seq_puts(m, "# Snapshot commands:\n"); if (iter->cpu_file == RING_BUFFER_ALL_CPUS) show_snapshot_main_help(m); else show_snapshot_percpu_help(m);}
static int tracing_snapshot_open(struct inode *inode, struct file *file){ struct trace_array *tr = inode->i_private; struct trace_iterator *iter; struct seq_file *m; int ret;
ret = tracing_check_open_get_tr(tr); if (ret) return ret;
if (file->f_mode & FMODE_READ) { iter = __tracing_open(inode, file, true); if (IS_ERR(iter)) ret = PTR_ERR(iter); } else { /* Writes still need the seq_file to hold the private data */ ret = -ENOMEM; m = kzalloc_obj(*m); if (!m) goto out; iter = kzalloc_obj(*iter); if (!iter) { kfree(m); goto out; } ret = 0;
iter->tr = tr; iter->array_buffer = &tr->snapshot_buffer; iter->cpu_file = tracing_get_cpu(inode); m->private = iter; file->private_data = m; }out: if (ret < 0) trace_array_put(tr);
return ret;}
static void tracing_swap_cpu_buffer(void *tr){ update_max_tr_single((struct trace_array *)tr, current, smp_processor_id());}
static ssize_ttracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos){ struct seq_file *m = filp->private_data; struct trace_iterator *iter = m->private; struct trace_array *tr = iter->tr; unsigned long val; int ret;
ret = tracing_update_buffers(tr); if (ret < 0) return ret;
ret = kstrtoul_from_user(ubuf, cnt, 10, &val); if (ret) return ret;
guard(mutex)(&trace_types_lock);
if (tracer_uses_snapshot(tr->current_trace)) return -EBUSY;
local_irq_disable(); arch_spin_lock(&tr->max_lock); if (tr->cond_snapshot) ret = -EBUSY; arch_spin_unlock(&tr->max_lock); local_irq_enable(); if (ret) return ret;
switch (val) { case 0: if (iter->cpu_file != RING_BUFFER_ALL_CPUS) return -EINVAL; if (tr->allocated_snapshot) free_snapshot(tr); break; case 1:/* Only allow per-cpu swap if the ring buffer supports it */#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP if (iter->cpu_file != RING_BUFFER_ALL_CPUS) return -EINVAL;#endif if (tr->allocated_snapshot) ret = resize_buffer_duplicate_size(&tr->snapshot_buffer, &tr->array_buffer, iter->cpu_file);
ret = tracing_arm_snapshot_locked(tr); if (ret) return ret;
/* Now, we're going to swap */ if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { local_irq_disable(); update_max_tr(tr, current, smp_processor_id(), NULL); local_irq_enable(); } else { smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, (void *)tr, 1); } tracing_disarm_snapshot(tr); break; default: if (tr->allocated_snapshot) { if (iter->cpu_file == RING_BUFFER_ALL_CPUS) tracing_reset_online_cpus(&tr->snapshot_buffer); else tracing_reset_cpu(&tr->snapshot_buffer, iter->cpu_file); } break; }
if (ret >= 0) { *ppos += cnt; ret = cnt; }
return ret;}
static int tracing_snapshot_release(struct inode *inode, struct file *file){ struct seq_file *m = file->private_data; int ret;
ret = tracing_release(inode, file);
if (file->f_mode & FMODE_READ) return ret;
/* If write only, the seq_file is just a stub */ if (m) kfree(m->private); kfree(m);
return 0;}
static int snapshot_raw_open(struct inode *inode, struct file *filp){ struct ftrace_buffer_info *info; int ret;
/* The following checks for tracefs lockdown */ ret = tracing_buffers_open(inode, filp); if (ret < 0) return ret;
info = filp->private_data;
if (tracer_uses_snapshot(info->iter.trace)) { tracing_buffers_release(inode, filp); return -EBUSY; }
info->iter.snapshot = true; info->iter.array_buffer = &info->iter.tr->snapshot_buffer;
return ret;}
const struct file_operations snapshot_fops = { .open = tracing_snapshot_open, .read = seq_read, .write = tracing_snapshot_write, .llseek = tracing_lseek, .release = tracing_snapshot_release,};
const struct file_operations snapshot_raw_fops = { .open = snapshot_raw_open, .read = tracing_buffers_read, .release = tracing_buffers_release, .splice_read = tracing_buffers_splice_read,};
#ifdef CONFIG_TRACER_MAX_TRACEstatic ssize_ttracing_max_lat_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos){ struct trace_array *tr = filp->private_data;
return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos);}
static ssize_ttracing_max_lat_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos){ struct trace_array *tr = filp->private_data;
return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos);}
static const struct file_operations tracing_max_lat_fops = { .open = tracing_open_generic_tr, .read = tracing_max_lat_read, .write = tracing_max_lat_write, .llseek = generic_file_llseek, .release = tracing_release_generic_tr,};#endif /* CONFIG_TRACER_MAX_TRACE */
int get_snapshot_map(struct trace_array *tr){ int err = 0;
/* * Called with mmap_lock held. lockdep would be unhappy if we would now * take trace_types_lock. Instead use the specific * snapshot_trigger_lock. */ spin_lock(&tr->snapshot_trigger_lock);
if (tr->snapshot || tr->mapped == UINT_MAX) err = -EBUSY; else tr->mapped++;
spin_unlock(&tr->snapshot_trigger_lock);
/* Wait for update_max_tr() to observe iter->tr->mapped */ if (tr->mapped == 1) synchronize_rcu();
return err;
}
void put_snapshot_map(struct trace_array *tr){ spin_lock(&tr->snapshot_trigger_lock); if (!WARN_ON(!tr->mapped)) tr->mapped--; spin_unlock(&tr->snapshot_trigger_lock);}
#ifdef CONFIG_DYNAMIC_FTRACEstatic voidftrace_snapshot(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data){ tracing_snapshot_instance(tr);}
static voidftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data){ struct ftrace_func_mapper *mapper = data; long *count = NULL;
if (mapper) count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
if (count) {
if (*count <= 0) return;
(*count)--; }
tracing_snapshot_instance(tr);}
static intftrace_snapshot_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data){ struct ftrace_func_mapper *mapper = data; long *count = NULL;
seq_printf(m, "%ps:", (void *)ip);
seq_puts(m, "snapshot");
if (mapper) count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
if (count) seq_printf(m, ":count=%ld\n", *count); else seq_puts(m, ":unlimited\n");
return 0;}
static intftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, unsigned long ip, void *init_data, void **data){ struct ftrace_func_mapper *mapper = *data;
if (!mapper) { mapper = allocate_ftrace_func_mapper(); if (!mapper) return -ENOMEM; *data = mapper; }
return ftrace_func_mapper_add_ip(mapper, ip, init_data);}
static voidftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, unsigned long ip, void *data){ struct ftrace_func_mapper *mapper = data;
if (!ip) { if (!mapper) return; free_ftrace_func_mapper(mapper, NULL); return; }
ftrace_func_mapper_remove_ip(mapper, ip);}
static struct ftrace_probe_ops snapshot_probe_ops = { .func = ftrace_snapshot, .print = ftrace_snapshot_print,};
static struct ftrace_probe_ops snapshot_count_probe_ops = { .func = ftrace_count_snapshot, .print = ftrace_snapshot_print, .init = ftrace_snapshot_init, .free = ftrace_snapshot_free,};
static intftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable){ struct ftrace_probe_ops *ops; void *count = (void *)-1; char *number; int ret;
if (!tr) return -ENODEV;
/* hash funcs only work with set_ftrace_filter */ if (!enable) return -EINVAL;
ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
if (glob[0] == '!') { ret = unregister_ftrace_function_probe_func(glob+1, tr, ops); if (!ret) tracing_disarm_snapshot(tr);
return ret; }
if (!param) goto out_reg;
number = strsep(¶m, ":");
if (!strlen(number)) goto out_reg;
/* * We use the callback data field (which is a pointer) * as our counter. */ ret = kstrtoul(number, 0, (unsigned long *)&count); if (ret) return ret;
out_reg: ret = tracing_arm_snapshot(tr); if (ret < 0) return ret;
ret = register_ftrace_function_probe(glob, tr, ops, count); if (ret < 0) tracing_disarm_snapshot(tr);
return ret < 0 ? ret : 0;}
static struct ftrace_func_command ftrace_snapshot_cmd = { .name = "snapshot", .func = ftrace_trace_snapshot_callback,};
__init int register_snapshot_cmd(void){ return register_ftrace_command(&ftrace_snapshot_cmd);}#endif /* CONFIG_DYNAMIC_FTRACE */
int trace_allocate_snapshot(struct trace_array *tr, int size){ int ret;
/* Fix mapped buffer trace arrays do not have snapshot buffers */ if (tr->range_addr_start) return 0;
/* allocate_snapshot can only be true during system boot */ ret = allocate_trace_buffer(tr, &tr->snapshot_buffer, allocate_snapshot ? size : 1); if (ret < 0) return -ENOMEM;
tr->allocated_snapshot = allocate_snapshot;
allocate_snapshot = false; return 0;}
__init static bool tr_needs_alloc_snapshot(const char *name){ char *test; int len = strlen(name); bool ret;
if (!boot_snapshot_index) return false;
if (strncmp(name, boot_snapshot_info, len) == 0 && boot_snapshot_info[len] == '\t') return true;
test = kmalloc(strlen(name) + 3, GFP_KERNEL); if (!test) return false;
sprintf(test, "\t%s\t", name); ret = strstr(boot_snapshot_info, test) == NULL; kfree(test); return ret;}
__init void do_allocate_snapshot(const char *name){ if (!tr_needs_alloc_snapshot(name)) return;
/* * When allocate_snapshot is set, the next call to * allocate_trace_buffers() (called by trace_array_get_by_name()) * will allocate the snapshot buffer. That will also clear * this flag. */ allocate_snapshot = true;}
void __init ftrace_boot_snapshot(void){ struct trace_array *tr;
if (!snapshot_at_boot) return;
list_for_each_entry(tr, &ftrace_trace_arrays, list) { if (!tr->allocated_snapshot) continue;
tracing_snapshot_instance(tr); trace_array_puts(tr, "** Boot snapshot taken **\n"); }}