I ported the latency tracer to 2.6.16 and got this 13ms latency within a
few hours. This is a regression from 2.6.15.
It appears that RCU can invoke ipv4_dst_destroy thousands of times in a
single batch.
preemption latency trace v1.1.5 on 2.6.16-rc1
--------------------------------------------------------------------
latency: 13805 us, #14352/14352, CPU#0 | (M:rt VP:0, KP:0, SP:0 HP:0)
-----------------
| task: gtk-gnutella-3116 (uid:1000 nice:0 policy:0 rt_prio:0)
-----------------
_------=> CPU#
/ _-----=> irqs-off
| / _----=> need-resched
|| / _---=> hardirq/softirq
||| / _--=> preempt-depth
|||| /
||||| delay
cmd pid ||||| time | caller
\ / ||||| \ | /
<idle>-0 0d.s2 1us : __trace_start_sched_wakeup (try_to_wake_up)
<idle>-0 0d.s2 2us : __trace_start_sched_wakeup <<...>-3116> (73 0)
<idle>-0 0d.s. 3us : wake_up_process (process_timeout)
<idle>-0 0d.s. 4us : net_rx_action (__do_softirq)
<idle>-0 0d.s. 5us : process_backlog (net_rx_action)
<idle>-0 0d.s. 6us : netif_receive_skb (process_backlog)
<idle>-0 0d.s1 7us : packet_rcv_spkt (netif_receive_skb)
<idle>-0 0d.s1 7us : skb_clone (packet_rcv_spkt)
<idle>-0 0d.s1 8us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s1 9us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s2 10us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s1 12us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s1 12us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s1 13us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s1 14us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s1 14us : ip_rcv (netif_receive_skb)
<idle>-0 0d.s1 15us : ip_route_input (ip_rcv)
<idle>-0 0d.s1 16us : rt_hash_code (ip_route_input)
<idle>-0 0d.s1 17us : memset (ip_route_input)
<idle>-0 0d.s1 18us : fn_hash_lookup (ip_route_input)
<idle>-0 0d.s2 19us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 20us : fib_validate_source (ip_route_input)
<idle>-0 0d.s1 21us : memset (fib_validate_source)
<idle>-0 0d.s1 22us : fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s1 23us : fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s2 24us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 25us : __fib_res_prefsrc (fib_validate_source)
<idle>-0 0d.s1 26us : inet_select_addr (__fib_res_prefsrc)
<idle>-0 0d.s1 27us : dst_alloc (ip_route_input)
<idle>-0 0d.s1 28us : rt_garbage_collect (dst_alloc)
<idle>-0 0d.s1 28us+: kmem_cache_alloc (dst_alloc)
<idle>-0 0d.s1 31us : rt_hash_code (ip_route_input)
<idle>-0 0d.s1 32us : rt_intern_hash (ip_route_input)
<idle>-0 0d.s1 32us : local_bh_enable (rt_intern_hash)
<idle>-0 0d.s1 33us : ip_local_deliver (ip_rcv)
<idle>-0 0d.s2 34us : tcp_v4_rcv (ip_local_deliver)
<idle>-0 0d.s2 35us : __skb_checksum_complete (tcp_v4_rcv)
<idle>-0 0d.s2 36us+: skb_checksum (__skb_checksum_complete)
<idle>-0 0d.s3 39us : tcp_v4_do_rcv (tcp_v4_rcv)
<idle>-0 0d.s3 39us : tcp_rcv_state_process (tcp_v4_do_rcv)
<idle>-0 0d.s3 40us+: tcp_parse_options (tcp_rcv_state_process)
<idle>-0 0d.s3 44us : tcp_ack (tcp_rcv_state_process)
<idle>-0 0d.s3 46us+: tcp_sync_mss (tcp_ack)
<idle>-0 0d.s3 49us : __kfree_skb (tcp_ack)
<idle>-0 0d.s3 50us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s3 50us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s3 51us : kfree (skb_release_data)
<idle>-0 0d.s3 52us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s3 54us : tcp_ack_saw_tstamp (tcp_ack)
<idle>-0 0d.s3 54us+: tcp_rtt_estimator (tcp_ack_saw_tstamp)
<idle>-0 0d.s3 58us : tcp_sync_mss (tcp_rcv_state_process)
<idle>-0 0d.s3 59us : tcp_initialize_rcv_mss (tcp_rcv_state_process)
<idle>-0 0d.s3 60us : inet_sk_rebuild_header (tcp_rcv_state_process)
<idle>-0 0d.s3 61us : tcp_init_metrics (tcp_rcv_state_process)
<idle>-0 0d.s3 63us : tcp_init_cwnd (tcp_init_metrics)
<idle>-0 0d.s3 64us : tcp_init_congestion_control (tcp_rcv_state_process)
<idle>-0 0d.s3 65us : bictcp_init (tcp_init_congestion_control)
<idle>-0 0d.s3 66us+: tcp_init_buffer_space (tcp_rcv_state_process)
<idle>-0 0d.s3 69us : sock_def_wakeup (tcp_rcv_state_process)
<idle>-0 0d.s4 70us : __wake_up (sock_def_wakeup)
<idle>-0 0d.s5 71us : __wake_up_common (__wake_up)
<idle>-0 0d.s5 72us : default_wake_function (__wake_up_common)
<idle>-0 0d.s5 73us : try_to_wake_up (default_wake_function)
<idle>-0 0d.s6 74us : sched_clock (try_to_wake_up)
<idle>-0 0d.s6 75us : recalc_task_prio (try_to_wake_up)
<idle>-0 0d.s6 76us : effective_prio (recalc_task_prio)
<idle>-0 0d.s6 77us : try_to_wake_up <<...>-2854> (73 1)
<idle>-0 0d.s6 78us : enqueue_task (try_to_wake_up)
<idle>-0 0d.s6 78us : __trace_start_sched_wakeup (try_to_wake_up)
<idle>-0 0d.s3 80us : tcp_send_ack (tcp_rcv_state_process)
<idle>-0 0d.s3 81us : __alloc_skb (tcp_send_ack)
<idle>-0 0d.s3 82us : kmem_cache_alloc (__alloc_skb)
<idle>-0 0d.s3 82us : __kmalloc (__alloc_skb)
<idle>-0 0d.s3 84us : tcp_transmit_skb (tcp_send_ack)
<idle>-0 0d.s3 85us : __tcp_select_window (tcp_transmit_skb)
<idle>-0 0d.s3 86us : tcp_v4_send_check (tcp_transmit_skb)
<idle>-0 0d.s3 87us : ip_queue_xmit (tcp_transmit_skb)
<idle>-0 0d.s3 88us : ip_output (ip_queue_xmit)
<idle>-0 0d.s3 90us : local_bh_enable (ip_output)
<idle>-0 0d.s3 91us : dev_queue_xmit (ip_output)
<idle>-0 0d.s4 91us : pfifo_fast_enqueue (dev_queue_xmit)
<idle>-0 0d.s4 92us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 93us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s4 94us : dev_queue_xmit_nit (qdisc_restart)
<idle>-0 0d.s5 95us : skb_clone (dev_queue_xmit_nit)
<idle>-0 0d.s5 95us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s5 97us : packet_rcv_spkt (dev_queue_xmit_nit)
<idle>-0 0d.s5 98us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s6 99us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s5 100us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s5 101us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s5 101us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s5 102us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s4 103us : rhine_start_tx (qdisc_restart)
<idle>-0 0d.s5 104us : ioread8 (rhine_start_tx)
<idle>-0 0d.s5 105us : iowrite8 (rhine_start_tx)
<idle>-0 0d.s5 106us : ioread8 (rhine_start_tx)
<idle>-0 0d.s4 108us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 108us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s3 109us : local_bh_enable (dev_queue_xmit)
<idle>-0 0d.s3 110us : tcp_urg (tcp_rcv_state_process)
<idle>-0 0d.s3 111us : __kfree_skb (tcp_rcv_state_process)
<idle>-0 0d.s3 111us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s3 112us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s3 113us : kfree (skb_release_data)
<idle>-0 0d.s3 114us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.H3 116us : do_IRQ (c012ae3b b 0)
<idle>-0 0d.h. 117us : __do_IRQ (do_IRQ)
<idle>-0 0d.h1 117us+: mask_and_ack_8259A (__do_IRQ)
<idle>-0 0d.h. 122us : handle_IRQ_event (__do_IRQ)
<idle>-0 0d.h. 123us : usb_hcd_irq (handle_IRQ_event)
<idle>-0 0d.h. 124us : uhci_irq (usb_hcd_irq)
<idle>-0 0d.h. 125us : rhine_interrupt (handle_IRQ_event)
<idle>-0 0d.h. 125us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 127us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 128us : iowrite16 (rhine_interrupt)
<idle>-0 0d.h. 128us : ioread8 (rhine_interrupt)
<idle>-0 0d.h1 130us : raise_softirq_irqoff (rhine_interrupt)
<idle>-0 0d.h. 131us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 132us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 133us : via_driver_irq_handler (handle_IRQ_event)
<idle>-0 0d.h1 135us : note_interrupt (__do_IRQ)
<idle>-0 0d.h1 135us : end_8259A_irq (__do_IRQ)
<idle>-0 0d.h1 136us+: enable_8259A_irq (end_8259A_irq)
<idle>-0 0d.H3 138us : irq_exit (do_IRQ)
<idle>-0 0d.s3 139us < (2097760)
<idle>-0 0d.s3 139us : tcp_current_mss (tcp_rcv_state_process)
<idle>-0 0d.s3 140us : __tcp_push_pending_frames (tcp_rcv_state_process)
<idle>-0 0d.s3 141us : tcp_check_space (tcp_rcv_state_process)
<idle>-0 0d.s. 143us : tasklet_action (__do_softirq)
<idle>-0 0d.s. 143us : rcu_process_callbacks (tasklet_action)
<idle>-0 0d.s. 144us : __rcu_process_callbacks (rcu_process_callbacks)
<idle>-0 0d.s. 145us : __rcu_process_callbacks (rcu_process_callbacks)
<idle>-0 0d.s. 146us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 147us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 148us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 149us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 150us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 151us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 151us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 152us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 153us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 154us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 154us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 155us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 156us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 157us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 158us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 158us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 159us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 160us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 161us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 162us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 163us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 163us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 164us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 165us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 166us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 166us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 167us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 168us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 169us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 170us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 170us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 171us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 172us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 173us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 174us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 174us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 175us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 176us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 177us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 178us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 179us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 179us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 180us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 181us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 182us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 182us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 183us : ipv4_dst_destroy (dst_destroy)
[ etc - zillions of dst_rcu_free()s deleted ]
<idle>-0 0d.s. 13403us : dst_rcu_free (__rcu_process_callbacks)
<idle>-0 0d.s. 13403us : dst_destroy (dst_rcu_free)
<idle>-0 0d.s. 13404us : ipv4_dst_destroy (dst_destroy)
<idle>-0 0d.s. 13405us : kmem_cache_free (dst_destroy)
<idle>-0 0d.s. 13406us : run_timer_softirq (__do_softirq)
<idle>-0 0d.s. 13407us : hrtimer_run_queues (run_timer_softirq)
<idle>-0 0d.s. 13408us : ktime_get_real (hrtimer_run_queues)
<idle>-0 0d.s. 13409us : getnstimeofday (ktime_get_real)
<idle>-0 0d.s. 13410us : do_gettimeofday (getnstimeofday)
<idle>-0 0d.s. 13410us+: get_offset_tsc (do_gettimeofday)
<idle>-0 0d.s. 13413us : ktime_get (hrtimer_run_queues)
<idle>-0 0d.s. 13413us : ktime_get_ts (ktime_get)
<idle>-0 0d.s. 13414us : getnstimeofday (ktime_get_ts)
<idle>-0 0d.s. 13414us : do_gettimeofday (getnstimeofday)
<idle>-0 0d.s. 13415us : get_offset_tsc (do_gettimeofday)
<idle>-0 0d.s. 13416us+: set_normalized_timespec (ktime_get_ts)
<idle>-0 0d.s. 13419us : net_tx_action (__do_softirq)
<idle>-0 0d.s. 13420us : __kfree_skb (net_tx_action)
<idle>-0 0d.s. 13421us : sock_wfree (__kfree_skb)
<idle>-0 0d.s. 13422us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s. 13423us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s. 13424us : kfree (skb_release_data)
<idle>-0 0d.s. 13425us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s. 13427us : __kfree_skb (net_tx_action)
<idle>-0 0d.s. 13428us : sock_wfree (__kfree_skb)
<idle>-0 0d.s. 13429us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s. 13430us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s. 13431us : kfree (skb_release_data)
<idle>-0 0d.s. 13432us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s. 13433us : net_rx_action (__do_softirq)
<idle>-0 0d.s. 13434us : process_backlog (net_rx_action)
<idle>-0 0d.s. 13435us+: netif_receive_skb (process_backlog)
<idle>-0 0d.s1 13437us : packet_rcv_spkt (netif_receive_skb)
<idle>-0 0d.s1 13438us : skb_clone (packet_rcv_spkt)
<idle>-0 0d.s1 13439us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s1 13440us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s2 13442us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s1 13444us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s1 13444us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s1 13445us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s1 13446us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s1 13446us : ip_rcv (netif_receive_skb)
<idle>-0 0d.s1 13448us : ip_route_input (ip_rcv)
<idle>-0 0d.s1 13449us+: rt_hash_code (ip_route_input)
<idle>-0 0d.s1 13451us : memset (ip_route_input)
<idle>-0 0d.s1 13452us+: fn_hash_lookup (ip_route_input)
<idle>-0 0d.s2 13454us+: fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 13457us : fib_validate_source (ip_route_input)
<idle>-0 0d.s1 13458us : memset (fib_validate_source)
<idle>-0 0d.s1 13459us+: fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s1 13461us : fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s2 13463us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 13464us : __fib_res_prefsrc (fib_validate_source)
<idle>-0 0d.s1 13465us : inet_select_addr (__fib_res_prefsrc)
<idle>-0 0d.s1 13466us : dst_alloc (ip_route_input)
<idle>-0 0d.s1 13467us+: kmem_cache_alloc (dst_alloc)
<idle>-0 0d.s1 13469us : rt_hash_code (ip_route_input)
<idle>-0 0d.s1 13470us : rt_intern_hash (ip_route_input)
<idle>-0 0d.s1 13471us : local_bh_enable (rt_intern_hash)
<idle>-0 0d.s1 13472us : ip_local_deliver (ip_rcv)
<idle>-0 0d.s2 13473us+: tcp_v4_rcv (ip_local_deliver)
<idle>-0 0d.s3 13477us : tcp_v4_do_rcv (tcp_v4_rcv)
<idle>-0 0d.s3 13477us : tcp_rcv_established (tcp_v4_do_rcv)
<idle>-0 0d.s3 13479us : __tcp_checksum_complete_user (tcp_rcv_established)
<idle>-0 0d.s3 13479us : __skb_checksum_complete (__tcp_checksum_complete_user)
<idle>-0 0d.s3 13480us+: skb_checksum (__skb_checksum_complete)
<idle>-0 0d.s3 13489us : tcp_event_data_recv (tcp_rcv_established)
<idle>-0 0d.s3 13491us : tcp_incr_quickack (tcp_event_data_recv)
<idle>-0 0d.s3 13492us : __tcp_ack_snd_check (tcp_rcv_established)
<idle>-0 0d.s3 13493us : tcp_send_ack (__tcp_ack_snd_check)
<idle>-0 0d.s3 13494us : __alloc_skb (tcp_send_ack)
<idle>-0 0d.s3 13494us : kmem_cache_alloc (__alloc_skb)
<idle>-0 0d.s3 13495us : __kmalloc (__alloc_skb)
<idle>-0 0d.s3 13496us : tcp_transmit_skb (tcp_send_ack)
<idle>-0 0d.s3 13498us : __tcp_select_window (tcp_transmit_skb)
<idle>-0 0d.s3 13499us : tcp_v4_send_check (tcp_transmit_skb)
<idle>-0 0d.s3 13500us : ip_queue_xmit (tcp_transmit_skb)
<idle>-0 0d.s3 13501us : ipv4_dst_check (ip_queue_xmit)
<idle>-0 0d.s3 13502us : memset (ip_queue_xmit)
<idle>-0 0d.s3 13504us : ip_route_output_flow (ip_queue_xmit)
<idle>-0 0d.s3 13505us : __ip_route_output_key (ip_route_output_flow)
<idle>-0 0d.s3 13506us : rt_hash_code (__ip_route_output_key)
<idle>-0 0d.s3 13507us : local_bh_enable (__ip_route_output_key)
<idle>-0 0d.s3 13508us : memset (__ip_route_output_key)
<idle>-0 0d.s3 13509us : ip_dev_find (__ip_route_output_key)
<idle>-0 0d.s3 13510us : memset (ip_dev_find)
<idle>-0 0d.s3 13511us : fn_hash_lookup (ip_dev_find)
<idle>-0 0d.s4 13512us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s3 13514us : fn_hash_lookup (__ip_route_output_key)
<idle>-0 0d.s3 13515us : fn_hash_lookup (__ip_route_output_key)
<idle>-0 0d.s4 13516us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s3 13518us+: fn_hash_select_default (__ip_route_output_key)
<idle>-0 0d.s3 13520us : dst_alloc (__ip_route_output_key)
<idle>-0 0d.s3 13521us+: kmem_cache_alloc (dst_alloc)
<idle>-0 0d.s3 13524us+: rt_set_nexthop (__ip_route_output_key)
<idle>-0 0d.s3 13526us : rt_hash_code (__ip_route_output_key)
<idle>-0 0d.s3 13527us : rt_intern_hash (__ip_route_output_key)
<idle>-0 0d.s4 13528us : arp_bind_neighbour (rt_intern_hash)
<idle>-0 0d.s4 13529us : neigh_lookup (arp_bind_neighbour)
<idle>-0 0d.s4 13530us+: arp_hash (neigh_lookup)
<idle>-0 0d.s5 13532us : memcmp (neigh_lookup)
<idle>-0 0d.s4 13534us : local_bh_enable (neigh_lookup)
<idle>-0 0d.s3 13535us+: local_bh_enable (rt_intern_hash)
<idle>-0 0d.s3 13537us : ip_output (ip_queue_xmit)
<idle>-0 0d.s3 13538us : neigh_resolve_output (ip_output)
<idle>-0 0d.s4 13540us+: eth_header (neigh_resolve_output)
<idle>-0 0d.H4 13543us : do_IRQ (c01f6117 0 0)
<idle>-0 0d.h. 13544us : __do_IRQ (do_IRQ)
<idle>-0 0d.h1 13545us+: mask_and_ack_8259A (__do_IRQ)
<idle>-0 0d.h. 13549us : handle_IRQ_event (__do_IRQ)
<idle>-0 0d.h. 13550us : timer_interrupt (handle_IRQ_event)
<idle>-0 0d.h1 13551us+: mark_offset_tsc (timer_interrupt)
<idle>-0 0d.h1 13557us : do_timer (timer_interrupt)
<idle>-0 0d.h1 13558us : update_process_times (timer_interrupt)
<idle>-0 0d.h1 13558us : account_system_time (update_process_times)
<idle>-0 0d.h1 13559us : acct_update_integrals (account_system_time)
<idle>-0 0d.h1 13560us : run_local_timers (update_process_times)
<idle>-0 0d.h1 13561us : raise_softirq (run_local_timers)
<idle>-0 0d.h1 13561us : rcu_pending (update_process_times)
<idle>-0 0d.h1 13562us : __rcu_pending (rcu_pending)
<idle>-0 0d.h1 13563us : __rcu_pending (rcu_pending)
<idle>-0 0d.h1 13563us : scheduler_tick (update_process_times)
<idle>-0 0d.h1 13564us : sched_clock (scheduler_tick)
<idle>-0 0d.h1 13565us : run_posix_cpu_timers (update_process_times)
<idle>-0 0d.h1 13566us : smp_local_timer_interrupt (timer_interrupt)
<idle>-0 0d.h1 13566us : profile_tick (smp_local_timer_interrupt)
<idle>-0 0d.h1 13567us : profile_hit (profile_tick)
<idle>-0 0d.h1 13568us : note_interrupt (__do_IRQ)
<idle>-0 0d.h1 13569us : end_8259A_irq (__do_IRQ)
<idle>-0 0d.h1 13569us : enable_8259A_irq (end_8259A_irq)
<idle>-0 0d.H4 13571us : irq_exit (do_IRQ)
<idle>-0 0d.s4 13572us < (2097760)
<idle>-0 0d.s3 13573us : local_bh_enable (neigh_resolve_output)
<idle>-0 0d.s3 13574us : dev_queue_xmit (neigh_resolve_output)
<idle>-0 0d.s4 13576us : pfifo_fast_enqueue (dev_queue_xmit)
<idle>-0 0d.s4 13577us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 13578us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s4 13579us : dev_queue_xmit_nit (qdisc_restart)
<idle>-0 0d.s5 13579us : skb_clone (dev_queue_xmit_nit)
<idle>-0 0d.s5 13580us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s5 13581us : packet_rcv_spkt (dev_queue_xmit_nit)
<idle>-0 0d.s5 13582us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s6 13583us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s5 13584us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s5 13585us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s5 13585us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s5 13586us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s4 13587us : rhine_start_tx (qdisc_restart)
<idle>-0 0d.s5 13589us : ioread8 (rhine_start_tx)
<idle>-0 0d.s5 13590us : iowrite8 (rhine_start_tx)
<idle>-0 0d.s5 13591us+: ioread8 (rhine_start_tx)
<idle>-0 0d.s4 13593us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 13594us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s3 13594us : local_bh_enable (dev_queue_xmit)
<idle>-0 0d.s3 13596us : sock_def_readable (tcp_rcv_established)
<idle>-0 0d.s4 13597us : __wake_up (sock_def_readable)
<idle>-0 0d.s5 13598us : __wake_up_common (__wake_up)
<idle>-0 0d.s5 13599us : ep_poll_callback (__wake_up_common)
<idle>-0 0d.s5 13601us : ep_poll_safewake (ep_poll_callback)
<idle>-0 0d.s5 13602us : __wake_up (ep_poll_safewake)
<idle>-0 0d.s6 13603us : __wake_up_common (__wake_up)
<idle>-0 0d.s6 13604us : default_wake_function (__wake_up_common)
<idle>-0 0d.s6 13605us+: try_to_wake_up (default_wake_function)
<idle>-0 0d.H5 13607us : do_IRQ (c0111d1e b 0)
<idle>-0 0d.h. 13608us : __do_IRQ (do_IRQ)
<idle>-0 0d.h1 13609us+: mask_and_ack_8259A (__do_IRQ)
<idle>-0 0d.h. 13614us : handle_IRQ_event (__do_IRQ)
<idle>-0 0d.h. 13615us : usb_hcd_irq (handle_IRQ_event)
<idle>-0 0d.h. 13616us : uhci_irq (usb_hcd_irq)
<idle>-0 0d.h. 13617us : rhine_interrupt (handle_IRQ_event)
<idle>-0 0d.h. 13618us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 13619us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 13621us : iowrite16 (rhine_interrupt)
<idle>-0 0d.h. 13621us : ioread8 (rhine_interrupt)
<idle>-0 0d.h1 13623us : raise_softirq_irqoff (rhine_interrupt)
<idle>-0 0d.h. 13624us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 13625us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 13627us+: via_driver_irq_handler (handle_IRQ_event)
<idle>-0 0d.h1 13629us : note_interrupt (__do_IRQ)
<idle>-0 0d.h1 13630us : end_8259A_irq (__do_IRQ)
<idle>-0 0d.h1 13631us : enable_8259A_irq (end_8259A_irq)
<idle>-0 0d.H5 13633us : irq_exit (do_IRQ)
<idle>-0 0d.s5 13633us+< (2097760)
<idle>-0 0d.s. 13635us : netif_receive_skb (process_backlog)
<idle>-0 0d.s1 13637us : packet_rcv_spkt (netif_receive_skb)
<idle>-0 0d.s1 13637us : skb_clone (packet_rcv_spkt)
<idle>-0 0d.s1 13638us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s1 13640us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s2 13640us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s1 13642us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s1 13642us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s1 13643us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s1 13644us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s1 13644us : ip_rcv (netif_receive_skb)
<idle>-0 0d.s1 13645us : ip_route_input (ip_rcv)
<idle>-0 0d.s1 13646us : rt_hash_code (ip_route_input)
<idle>-0 0d.s1 13647us : memset (ip_route_input)
<idle>-0 0d.s1 13648us : fn_hash_lookup (ip_route_input)
<idle>-0 0d.s2 13649us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 13650us : fib_validate_source (ip_route_input)
<idle>-0 0d.s1 13651us : memset (fib_validate_source)
<idle>-0 0d.s1 13652us : fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s1 13653us : fn_hash_lookup (fib_validate_source)
<idle>-0 0d.s2 13654us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s1 13655us : __fib_res_prefsrc (fib_validate_source)
<idle>-0 0d.s1 13656us : inet_select_addr (__fib_res_prefsrc)
<idle>-0 0d.s1 13657us : dst_alloc (ip_route_input)
<idle>-0 0d.s1 13657us : kmem_cache_alloc (dst_alloc)
<idle>-0 0d.s1 13659us : rt_hash_code (ip_route_input)
<idle>-0 0d.s1 13660us : rt_intern_hash (ip_route_input)
<idle>-0 0d.s1 13661us : local_bh_enable (rt_intern_hash)
<idle>-0 0d.s1 13662us : ip_local_deliver (ip_rcv)
<idle>-0 0d.s2 13663us+: tcp_v4_rcv (ip_local_deliver)
<idle>-0 0d.s3 13665us : tcp_v4_do_rcv (tcp_v4_rcv)
<idle>-0 0d.s3 13666us : tcp_rcv_established (tcp_v4_do_rcv)
<idle>-0 0d.s3 13667us : __tcp_checksum_complete_user (tcp_rcv_established)
<idle>-0 0d.s3 13667us : __skb_checksum_complete (__tcp_checksum_complete_user)
<idle>-0 0d.s3 13668us : skb_checksum (__skb_checksum_complete)
<idle>-0 0d.s3 13670us : tcp_event_data_recv (tcp_rcv_established)
<idle>-0 0d.s3 13671us : __tcp_ack_snd_check (tcp_rcv_established)
<idle>-0 0d.s3 13672us : tcp_send_ack (__tcp_ack_snd_check)
<idle>-0 0d.s3 13672us : __alloc_skb (tcp_send_ack)
<idle>-0 0d.s3 13673us : kmem_cache_alloc (__alloc_skb)
<idle>-0 0d.s3 13674us : __kmalloc (__alloc_skb)
<idle>-0 0d.s3 13675us : tcp_transmit_skb (tcp_send_ack)
<idle>-0 0d.s3 13676us : __tcp_select_window (tcp_transmit_skb)
<idle>-0 0d.s3 13677us : tcp_v4_send_check (tcp_transmit_skb)
<idle>-0 0d.s3 13678us : ip_queue_xmit (tcp_transmit_skb)
<idle>-0 0d.s3 13679us : ipv4_dst_check (ip_queue_xmit)
<idle>-0 0d.s3 13680us : memset (ip_queue_xmit)
<idle>-0 0d.s3 13681us : ip_route_output_flow (ip_queue_xmit)
<idle>-0 0d.s3 13682us : __ip_route_output_key (ip_route_output_flow)
<idle>-0 0d.s3 13682us : rt_hash_code (__ip_route_output_key)
<idle>-0 0d.s3 13683us : local_bh_enable (__ip_route_output_key)
<idle>-0 0d.s3 13684us : memset (__ip_route_output_key)
<idle>-0 0d.s3 13685us : ip_dev_find (__ip_route_output_key)
<idle>-0 0d.s3 13686us : memset (ip_dev_find)
<idle>-0 0d.s3 13687us : fn_hash_lookup (ip_dev_find)
<idle>-0 0d.s4 13688us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s3 13689us : fn_hash_lookup (__ip_route_output_key)
<idle>-0 0d.s3 13690us : fn_hash_lookup (__ip_route_output_key)
<idle>-0 0d.s4 13691us : fib_semantic_match (fn_hash_lookup)
<idle>-0 0d.s3 13692us : fn_hash_select_default (__ip_route_output_key)
<idle>-0 0d.s3 13693us : dst_alloc (__ip_route_output_key)
<idle>-0 0d.s3 13694us+: kmem_cache_alloc (dst_alloc)
<idle>-0 0d.s3 13696us : rt_set_nexthop (__ip_route_output_key)
<idle>-0 0d.s3 13697us : rt_hash_code (__ip_route_output_key)
<idle>-0 0d.s3 13698us : rt_intern_hash (__ip_route_output_key)
<idle>-0 0d.s4 13698us : arp_bind_neighbour (rt_intern_hash)
<idle>-0 0d.s4 13699us : neigh_lookup (arp_bind_neighbour)
<idle>-0 0d.s4 13700us : arp_hash (neigh_lookup)
<idle>-0 0d.s5 13701us : memcmp (neigh_lookup)
<idle>-0 0d.s4 13702us : local_bh_enable (neigh_lookup)
<idle>-0 0d.s3 13703us : local_bh_enable (rt_intern_hash)
<idle>-0 0d.s3 13704us : ip_output (ip_queue_xmit)
<idle>-0 0d.s3 13705us : neigh_resolve_output (ip_output)
<idle>-0 0d.s4 13706us : eth_header (neigh_resolve_output)
<idle>-0 0d.s3 13707us : local_bh_enable (neigh_resolve_output)
<idle>-0 0d.s3 13708us : dev_queue_xmit (neigh_resolve_output)
<idle>-0 0d.s4 13709us : pfifo_fast_enqueue (dev_queue_xmit)
<idle>-0 0d.s4 13710us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 13710us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s4 13711us : dev_queue_xmit_nit (qdisc_restart)
<idle>-0 0d.s5 13712us : skb_clone (dev_queue_xmit_nit)
<idle>-0 0d.s5 13713us : kmem_cache_alloc (skb_clone)
<idle>-0 0d.s5 13714us : packet_rcv_spkt (dev_queue_xmit_nit)
<idle>-0 0d.s5 13715us : strlcpy (packet_rcv_spkt)
<idle>-0 0d.s6 13716us : sk_run_filter (packet_rcv_spkt)
<idle>-0 0d.s5 13717us : __kfree_skb (packet_rcv_spkt)
<idle>-0 0d.s5 13718us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s5 13718us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s5 13719us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s4 13720us : rhine_start_tx (qdisc_restart)
<idle>-0 0d.s5 13721us : ioread8 (rhine_start_tx)
<idle>-0 0d.s5 13722us : iowrite8 (rhine_start_tx)
<idle>-0 0d.s5 13723us : ioread8 (rhine_start_tx)
<idle>-0 0d.s4 13725us : qdisc_restart (dev_queue_xmit)
<idle>-0 0d.s4 13725us : pfifo_fast_dequeue (qdisc_restart)
<idle>-0 0d.s3 13726us : local_bh_enable (dev_queue_xmit)
<idle>-0 0d.s3 13727us : sock_def_readable (tcp_rcv_established)
<idle>-0 0d.s4 13728us : __wake_up (sock_def_readable)
<idle>-0 0d.s5 13729us : __wake_up_common (__wake_up)
<idle>-0 0d.s5 13730us : ep_poll_callback (__wake_up_common)
<idle>-0 0d.s5 13732us : ep_poll_safewake (ep_poll_callback)
<idle>-0 0d.s5 13733us : __wake_up (ep_poll_safewake)
<idle>-0 0d.s6 13733us : __wake_up_common (__wake_up)
<idle>-0 0d.s6 13734us : default_wake_function (__wake_up_common)
<idle>-0 0d.s6 13735us+: try_to_wake_up (default_wake_function)
<idle>-0 0d.H5 13737us : do_IRQ (c0111d1e b 0)
<idle>-0 0d.h. 13738us : __do_IRQ (do_IRQ)
<idle>-0 0d.h1 13739us+: mask_and_ack_8259A (__do_IRQ)
<idle>-0 0d.h. 13743us : handle_IRQ_event (__do_IRQ)
<idle>-0 0d.h. 13744us : usb_hcd_irq (handle_IRQ_event)
<idle>-0 0d.h. 13745us : uhci_irq (usb_hcd_irq)
<idle>-0 0d.h. 13746us : rhine_interrupt (handle_IRQ_event)
<idle>-0 0d.h. 13746us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 13748us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 13749us : iowrite16 (rhine_interrupt)
<idle>-0 0d.h. 13750us : ioread8 (rhine_interrupt)
<idle>-0 0d.h1 13751us : raise_softirq_irqoff (rhine_interrupt)
<idle>-0 0d.h. 13752us : ioread16 (rhine_interrupt)
<idle>-0 0d.h. 13753us : ioread8 (rhine_interrupt)
<idle>-0 0d.h. 13754us : via_driver_irq_handler (handle_IRQ_event)
<idle>-0 0d.h1 13756us : note_interrupt (__do_IRQ)
<idle>-0 0d.h1 13756us : end_8259A_irq (__do_IRQ)
<idle>-0 0d.h1 13757us : enable_8259A_irq (end_8259A_irq)
<idle>-0 0d.H5 13759us : irq_exit (do_IRQ)
<idle>-0 0d.s5 13760us+< (2097760)
<idle>-0 0d.s. 13762us : tasklet_action (__do_softirq)
<idle>-0 0d.s. 13763us : rcu_process_callbacks (tasklet_action)
<idle>-0 0d.s. 13764us : __rcu_process_callbacks (rcu_process_callbacks)
<idle>-0 0d.s. 13765us : __rcu_process_callbacks (rcu_process_callbacks)
<idle>-0 0d.s. 13766us : run_timer_softirq (__do_softirq)
<idle>-0 0d.s. 13766us : hrtimer_run_queues (run_timer_softirq)
<idle>-0 0d.s. 13767us : ktime_get_real (hrtimer_run_queues)
<idle>-0 0d.s. 13768us : getnstimeofday (ktime_get_real)
<idle>-0 0d.s. 13768us : do_gettimeofday (getnstimeofday)
<idle>-0 0d.s. 13769us : get_offset_tsc (do_gettimeofday)
<idle>-0 0d.s. 13770us : ktime_get (hrtimer_run_queues)
<idle>-0 0d.s. 13771us : ktime_get_ts (ktime_get)
<idle>-0 0d.s. 13771us : getnstimeofday (ktime_get_ts)
<idle>-0 0d.s. 13772us : do_gettimeofday (getnstimeofday)
<idle>-0 0d.s. 13773us : get_offset_tsc (do_gettimeofday)
<idle>-0 0d.s. 13773us : set_normalized_timespec (ktime_get_ts)
<idle>-0 0d.s. 13775us : net_tx_action (__do_softirq)
<idle>-0 0d.s. 13775us : __kfree_skb (net_tx_action)
<idle>-0 0d.s. 13776us : sock_wfree (__kfree_skb)
<idle>-0 0d.s. 13777us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s. 13778us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s. 13778us : kfree (skb_release_data)
<idle>-0 0d.s. 13779us : kmem_cache_free (kfree_skbmem)
<idle>-0 0d.s. 13780us : __kfree_skb (net_tx_action)
<idle>-0 0d.s. 13780us : sock_wfree (__kfree_skb)
<idle>-0 0d.s. 13781us : kfree_skbmem (__kfree_skb)
<idle>-0 0d.s. 13782us : skb_release_data (kfree_skbmem)
<idle>-0 0d.s. 13782us : kfree (skb_release_data)
<idle>-0 0d.s. 13783us+: kmem_cache_free (kfree_skbmem)
<idle>-0 0dn.1 13785us < (2097760)
<idle>-0 0dn.. 13787us : schedule (cpu_idle)
<idle>-0 0dn.. 13788us : stop_trace (schedule)
<idle>-0 0dn.. 13789us : profile_hit (schedule)
<idle>-0 0dn.1 13789us+: sched_clock (schedule)
<idle>-0 0dn.2 13791us : recalc_task_prio (schedule)
<idle>-0 0dn.2 13793us : effective_prio (recalc_task_prio)
<idle>-0 0dn.2 13794us : requeue_task (schedule)
<...>-3116 0d..2 13797us+: __switch_to (schedule)
<...>-3116 0d..2 13801us : schedule <<idle>-0> (8c 73)
<...>-3116 0d..1 13802us : trace_stop_sched_switched (schedule)
<...>-3116 0d..2 13802us : trace_stop_sched_switched <<...>-3116> (73 0)
<...>-3116 0d..2 13804us : schedule (schedule)
Lee
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[Index of Archives]
[Kernel Newbies]
[Netfilter]
[Bugtraq]
[Photo]
[Stuff]
[Gimp]
[Yosemite News]
[MIPS Linux]
[ARM Linux]
[Linux Security]
[Linux RAID]
[Video 4 Linux]
[Linux for the blind]
[Linux Resources]
