024-8-tcp-tsq-move-tsq_flags-close-to-sk_wmem_alloc.patch 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. From 7aa5470c2c09265902b5e4289afa82e4e7c2987e Mon Sep 17 00:00:00 2001
  2. From: Eric Dumazet <edumazet@google.com>
  3. Date: Sat, 3 Dec 2016 11:14:57 -0800
  4. Subject: [PATCH 08/10] tcp: tsq: move tsq_flags close to sk_wmem_alloc
  5. tsq_flags being in the same cache line than sk_wmem_alloc
  6. makes a lot of sense. Both fields are changed from tcp_wfree()
  7. and more generally by various TSQ related functions.
  8. Prior patch made room in struct sock and added sk_tsq_flags,
  9. this patch deletes tsq_flags from struct tcp_sock.
  10. Signed-off-by: Eric Dumazet <edumazet@google.com>
  11. Signed-off-by: David S. Miller <davem@davemloft.net>
  12. ---
  13. include/linux/tcp.h | 1 -
  14. net/ipv4/tcp.c | 4 ++--
  15. net/ipv4/tcp_ipv4.c | 2 +-
  16. net/ipv4/tcp_output.c | 24 +++++++++++-------------
  17. net/ipv4/tcp_timer.c | 4 ++--
  18. net/ipv6/tcp_ipv6.c | 2 +-
  19. 6 files changed, 17 insertions(+), 20 deletions(-)
  20. --- a/include/linux/tcp.h
  21. +++ b/include/linux/tcp.h
  22. @@ -192,7 +192,6 @@ struct tcp_sock {
  23. u32 tsoffset; /* timestamp offset */
  24. struct list_head tsq_node; /* anchor in tsq_tasklet.head list */
  25. - unsigned long tsq_flags;
  26. /* Data for direct copy to user */
  27. struct {
  28. --- a/net/ipv4/tcp.c
  29. +++ b/net/ipv4/tcp.c
  30. @@ -665,9 +665,9 @@ static void tcp_push(struct sock *sk, in
  31. if (tcp_should_autocork(sk, skb, size_goal)) {
  32. /* avoid atomic op if TSQ_THROTTLED bit is already set */
  33. - if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
  34. + if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
  35. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
  36. - set_bit(TSQ_THROTTLED, &tp->tsq_flags);
  37. + set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
  38. }
  39. /* It is possible TX completion already happened
  40. * before we set TSQ_THROTTLED.
  41. --- a/net/ipv4/tcp_ipv4.c
  42. +++ b/net/ipv4/tcp_ipv4.c
  43. @@ -446,7 +446,7 @@ void tcp_v4_err(struct sk_buff *icmp_skb
  44. if (!sock_owned_by_user(sk)) {
  45. tcp_v4_mtu_reduced(sk);
  46. } else {
  47. - if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &tp->tsq_flags))
  48. + if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
  49. sock_hold(sk);
  50. }
  51. goto out;
  52. --- a/net/ipv4/tcp_output.c
  53. +++ b/net/ipv4/tcp_output.c
  54. @@ -772,14 +772,15 @@ static void tcp_tasklet_func(unsigned lo
  55. list_for_each_safe(q, n, &list) {
  56. tp = list_entry(q, struct tcp_sock, tsq_node);
  57. list_del(&tp->tsq_node);
  58. - clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  59. sk = (struct sock *)tp;
  60. + clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
  61. +
  62. if (!sk->sk_lock.owned &&
  63. - test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
  64. + test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
  65. bh_lock_sock(sk);
  66. if (!sock_owned_by_user(sk)) {
  67. - clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  68. + clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
  69. tcp_tsq_handler(sk);
  70. }
  71. bh_unlock_sock(sk);
  72. @@ -802,16 +803,15 @@ static void tcp_tasklet_func(unsigned lo
  73. */
  74. void tcp_release_cb(struct sock *sk)
  75. {
  76. - struct tcp_sock *tp = tcp_sk(sk);
  77. unsigned long flags, nflags;
  78. /* perform an atomic operation only if at least one flag is set */
  79. do {
  80. - flags = tp->tsq_flags;
  81. + flags = sk->sk_tsq_flags;
  82. if (!(flags & TCP_DEFERRED_ALL))
  83. return;
  84. nflags = flags & ~TCP_DEFERRED_ALL;
  85. - } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
  86. + } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
  87. if (flags & TCPF_TSQ_DEFERRED)
  88. tcp_tsq_handler(sk);
  89. @@ -883,7 +883,7 @@ void tcp_wfree(struct sk_buff *skb)
  90. if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
  91. goto out;
  92. - for (oval = READ_ONCE(tp->tsq_flags);; oval = nval) {
  93. + for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
  94. struct tsq_tasklet *tsq;
  95. bool empty;
  96. @@ -891,7 +891,7 @@ void tcp_wfree(struct sk_buff *skb)
  97. goto out;
  98. nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
  99. - nval = cmpxchg(&tp->tsq_flags, oval, nval);
  100. + nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
  101. if (nval != oval)
  102. continue;
  103. @@ -2136,7 +2136,7 @@ static bool tcp_small_queue_check(struct
  104. skb->prev == sk->sk_write_queue.next)
  105. return false;
  106. - set_bit(TSQ_THROTTLED, &tcp_sk(sk)->tsq_flags);
  107. + set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
  108. /* It is possible TX completion already happened
  109. * before we set TSQ_THROTTLED, so we must
  110. * test again the condition.
  111. @@ -2234,8 +2234,8 @@ static bool tcp_write_xmit(struct sock *
  112. unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
  113. break;
  114. - if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
  115. - clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  116. + if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
  117. + clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
  118. if (tcp_small_queue_check(sk, skb, 0))
  119. break;
  120. @@ -3542,8 +3542,6 @@ void __tcp_send_ack(struct sock *sk, u32
  121. /* We do not want pure acks influencing TCP Small Queues or fq/pacing
  122. * too much.
  123. * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
  124. - * We also avoid tcp_wfree() overhead (cache line miss accessing
  125. - * tp->tsq_flags) by using regular sock_wfree()
  126. */
  127. skb_set_tcp_pure_ack(buff);
  128. --- a/net/ipv4/tcp_timer.c
  129. +++ b/net/ipv4/tcp_timer.c
  130. @@ -326,7 +326,7 @@ static void tcp_delack_timer(unsigned lo
  131. inet_csk(sk)->icsk_ack.blocked = 1;
  132. __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED);
  133. /* deleguate our work to tcp_release_cb() */
  134. - if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
  135. + if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags))
  136. sock_hold(sk);
  137. }
  138. bh_unlock_sock(sk);
  139. @@ -609,7 +609,7 @@ static void tcp_write_timer(unsigned lon
  140. tcp_write_timer_handler(sk);
  141. } else {
  142. /* delegate our work to tcp_release_cb() */
  143. - if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
  144. + if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &sk->sk_tsq_flags))
  145. sock_hold(sk);
  146. }
  147. bh_unlock_sock(sk);
  148. --- a/net/ipv6/tcp_ipv6.c
  149. +++ b/net/ipv6/tcp_ipv6.c
  150. @@ -404,7 +404,7 @@ static void tcp_v6_err(struct sk_buff *s
  151. if (!sock_owned_by_user(sk))
  152. tcp_v6_mtu_reduced(sk);
  153. else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
  154. - &tp->tsq_flags))
  155. + &sk->sk_tsq_flags))
  156. sock_hold(sk);
  157. goto out;
  158. }