024-3-tcp-tsq-add-shortcut-in-tcp_tasklet_func.patch 2.2 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071
  1. From b223feb9de2a65c533ff95c08e834fa732906ea5 Mon Sep 17 00:00:00 2001
  2. From: Eric Dumazet <edumazet@google.com>
  3. Date: Sat, 3 Dec 2016 11:14:52 -0800
  4. Subject: [PATCH 03/10] tcp: tsq: add shortcut in tcp_tasklet_func()
  5. Under high stress, I've seen tcp_tasklet_func() consuming
  6. ~700 usec, handling ~150 tcp sockets.
  7. By setting TCP_TSQ_DEFERRED in tcp_wfree(), we give a chance
  8. for other cpus/threads entering tcp_write_xmit() to grab it,
  9. allowing tcp_tasklet_func() to skip sockets that already did
  10. an xmit cycle.
  11. In the future, we might give to ACK processing an increased
  12. budget to reduce even more tcp_tasklet_func() amount of work.
  13. Signed-off-by: Eric Dumazet <edumazet@google.com>
  14. Signed-off-by: David S. Miller <davem@davemloft.net>
  15. ---
  16. net/ipv4/tcp_output.c | 22 ++++++++++++----------
  17. 1 file changed, 12 insertions(+), 10 deletions(-)
  18. --- a/net/ipv4/tcp_output.c
  19. +++ b/net/ipv4/tcp_output.c
  20. @@ -772,19 +772,19 @@ static void tcp_tasklet_func(unsigned lo
  21. list_for_each_safe(q, n, &list) {
  22. tp = list_entry(q, struct tcp_sock, tsq_node);
  23. list_del(&tp->tsq_node);
  24. + clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  25. sk = (struct sock *)tp;
  26. - bh_lock_sock(sk);
  27. -
  28. - if (!sock_owned_by_user(sk)) {
  29. - tcp_tsq_handler(sk);
  30. - } else {
  31. - /* defer the work to tcp_release_cb() */
  32. - set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  33. + if (!sk->sk_lock.owned &&
  34. + test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
  35. + bh_lock_sock(sk);
  36. + if (!sock_owned_by_user(sk)) {
  37. + clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  38. + tcp_tsq_handler(sk);
  39. + }
  40. + bh_unlock_sock(sk);
  41. }
  42. - bh_unlock_sock(sk);
  43. - clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  44. sk_free(sk);
  45. }
  46. }
  47. @@ -889,7 +889,7 @@ void tcp_wfree(struct sk_buff *skb)
  48. if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
  49. goto out;
  50. - nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
  51. + nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
  52. nval = cmpxchg(&tp->tsq_flags, oval, nval);
  53. if (nval != oval)
  54. continue;
  55. @@ -2222,6 +2222,8 @@ static bool tcp_write_xmit(struct sock *
  56. unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
  57. break;
  58. + if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
  59. + clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  60. if (tcp_small_queue_check(sk, skb, 0))
  61. break;