asm-generic/io.h: Skip trace helpers if rwmmio events are disabled

Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git

kernel os linux

With `CONFIG_TRACE_MMIO_ACCESS=y`, the `{read,write}{b,w,l,q}{_relaxed}()`
mmio accessors unconditionally call `log_{post_}{read,write}_mmio()`
helpers, which in turn call the ftrace ops for `rwmmio` trace events

This adds a performance penalty per mmio accessor call, even when
`rwmmio` events are disabled at runtime (~80% overhead on local
measurement).

Guard these with `tracepoint_enabled()`.

Signed-off-by: Varad Gautam <varadgautam@google.com>
Fixes: 210031971cdd ("asm-generic/io: Add logging support for MMIO accessors")
Cc: stable@vger.kernel.org
Signed-off-by: Arnd Bergmann <arnd@arndb.de>

authored by

Varad Gautam and committed by

Arnd Bergmann 6 months ago 8327bd4f 8f5ae30d

+66 -32

1 changed file

expand all

include

asm-generic

io.h

+66 -32

include/asm-generic/io.h

··· 75 75 #if IS_ENABLED(CONFIG_TRACE_MMIO_ACCESS) && !(defined(__DISABLE_TRACE_MMIO__)) 76 76 #include <linux/tracepoint-defs.h> 77 77 78 + #define rwmmio_tracepoint_enabled(tracepoint) tracepoint_enabled(tracepoint) 78 79 DECLARE_TRACEPOINT(rwmmio_write); 79 80 DECLARE_TRACEPOINT(rwmmio_post_write); 80 81 DECLARE_TRACEPOINT(rwmmio_read); ··· 92 91 93 92 #else 94 93 94 + #define rwmmio_tracepoint_enabled(tracepoint) false 95 95 static inline void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr, 96 96 unsigned long caller_addr, unsigned long caller_addr0) {} 97 97 static inline void log_post_write_mmio(u64 val, u8 width, volatile void __iomem *addr, ··· 191 189 { 192 190 u8 val; 193 191 194 - log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); 192 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 193 + log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); 195 194 __io_br(); 196 195 val = __raw_readb(addr); 197 196 __io_ar(val); 198 - log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); 197 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 198 + log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); 199 199 return val; 200 200 } 201 201 #endif ··· 208 204 { 209 205 u16 val; 210 206 211 - log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); 207 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 208 + log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); 212 209 __io_br(); 213 210 val = __le16_to_cpu((__le16 __force)__raw_readw(addr)); 214 211 __io_ar(val); 215 - log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); 212 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 213 + log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); 216 214 return val; 217 215 } 218 216 #endif ··· 225 219 { 226 220 u32 val; 227 221 228 - log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); 222 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 223 + log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); 229 224 __io_br(); 230 225 val = __le32_to_cpu((__le32 __force)__raw_readl(addr)); 231 226 __io_ar(val); 232 - log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); 227 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 228 + log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); 233 229 return val; 234 230 } 235 231 #endif ··· 243 235 { 244 236 u64 val; 245 237 246 - log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); 238 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 239 + log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); 247 240 __io_br(); 248 241 val = __le64_to_cpu((__le64 __force)__raw_readq(addr)); 249 242 __io_ar(val); 250 - log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); 243 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 244 + log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); 251 245 return val; 252 246 } 253 247 #endif ··· 259 249 #define writeb writeb 260 250 static inline void writeb(u8 value, volatile void __iomem *addr) 261 251 { 262 - log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 252 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 253 + log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 263 254 __io_bw(); 264 255 __raw_writeb(value, addr); 265 256 __io_aw(); 266 - log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 257 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 258 + log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 267 259 } 268 260 #endif 269 261 ··· 273 261 #define writew writew 274 262 static inline void writew(u16 value, volatile void __iomem *addr) 275 263 { 276 - log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 264 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 265 + log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 277 266 __io_bw(); 278 267 __raw_writew((u16 __force)cpu_to_le16(value), addr); 279 268 __io_aw(); 280 - log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 269 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 270 + log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 281 271 } 282 272 #endif 283 273 ··· 287 273 #define writel writel 288 274 static inline void writel(u32 value, volatile void __iomem *addr) 289 275 { 290 - log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 276 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 277 + log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 291 278 __io_bw(); 292 279 __raw_writel((u32 __force)__cpu_to_le32(value), addr); 293 280 __io_aw(); 294 - log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 281 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 282 + log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 295 283 } 296 284 #endif 297 285 ··· 302 286 #define writeq writeq 303 287 static inline void writeq(u64 value, volatile void __iomem *addr) 304 288 { 305 - log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 289 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 290 + log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 306 291 __io_bw(); 307 292 __raw_writeq((u64 __force)__cpu_to_le64(value), addr); 308 293 __io_aw(); 309 - log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 294 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 295 + log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 310 296 } 311 297 #endif 312 298 #endif /* CONFIG_64BIT */ ··· 324 306 { 325 307 u8 val; 326 308 327 - log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); 309 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 310 + log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); 328 311 val = __raw_readb(addr); 329 - log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); 312 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 313 + log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); 330 314 return val; 331 315 } 332 316 #endif ··· 339 319 { 340 320 u16 val; 341 321 342 - log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); 322 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 323 + log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); 343 324 val = __le16_to_cpu((__le16 __force)__raw_readw(addr)); 344 - log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); 325 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 326 + log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); 345 327 return val; 346 328 } 347 329 #endif ··· 354 332 { 355 333 u32 val; 356 334 357 - log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); 335 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 336 + log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); 358 337 val = __le32_to_cpu((__le32 __force)__raw_readl(addr)); 359 - log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); 338 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 339 + log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); 360 340 return val; 361 341 } 362 342 #endif ··· 369 345 { 370 346 u64 val; 371 347 372 - log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); 348 + if (rwmmio_tracepoint_enabled(rwmmio_read)) 349 + log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); 373 350 val = __le64_to_cpu((__le64 __force)__raw_readq(addr)); 374 - log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); 351 + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) 352 + log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); 375 353 return val; 376 354 } 377 355 #endif ··· 382 356 #define writeb_relaxed writeb_relaxed 383 357 static inline void writeb_relaxed(u8 value, volatile void __iomem *addr) 384 358 { 385 - log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 359 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 360 + log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 386 361 __raw_writeb(value, addr); 387 - log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 362 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 363 + log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); 388 364 } 389 365 #endif 390 366 ··· 394 366 #define writew_relaxed writew_relaxed 395 367 static inline void writew_relaxed(u16 value, volatile void __iomem *addr) 396 368 { 397 - log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 369 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 370 + log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 398 371 __raw_writew((u16 __force)cpu_to_le16(value), addr); 399 - log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 372 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 373 + log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); 400 374 } 401 375 #endif 402 376 ··· 406 376 #define writel_relaxed writel_relaxed 407 377 static inline void writel_relaxed(u32 value, volatile void __iomem *addr) 408 378 { 409 - log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 379 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 380 + log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 410 381 __raw_writel((u32 __force)__cpu_to_le32(value), addr); 411 - log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 382 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 383 + log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); 412 384 } 413 385 #endif 414 386 ··· 418 386 #define writeq_relaxed writeq_relaxed 419 387 static inline void writeq_relaxed(u64 value, volatile void __iomem *addr) 420 388 { 421 - log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 389 + if (rwmmio_tracepoint_enabled(rwmmio_write)) 390 + log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 422 391 __raw_writeq((u64 __force)__cpu_to_le64(value), addr); 423 - log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 392 + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) 393 + log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); 424 394 } 425 395 #endif 426 396