b2ea46fe23
An overall throughput gain of 22 % for heavy TCP use over a single TX queue. Original patchset comment https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v4.13&id=3f4888adae7c1619b990d98a9b967536f71822b8 Signed-off-by: Pavel Kubelun <be.dissent@gmail.com>
48 lines
1.6 KiB
Diff
48 lines
1.6 KiB
Diff
From 408f0a6c21e124cc4f6c7aa370b38aa47e55428d Mon Sep 17 00:00:00 2001
|
|
From: Eric Dumazet <edumazet@google.com>
|
|
Date: Sat, 3 Dec 2016 11:14:51 -0800
|
|
Subject: [PATCH 02/10] tcp: tsq: remove one locked operation in tcp_wfree()
|
|
|
|
Instead of atomically clear TSQ_THROTTLED and atomically set TSQ_QUEUED
|
|
bits, use one cmpxchg() to perform a single locked operation.
|
|
|
|
Since the following patch will also set TCP_TSQ_DEFERRED here,
|
|
this cmpxchg() will make this addition free.
|
|
|
|
Signed-off-by: Eric Dumazet <edumazet@google.com>
|
|
Signed-off-by: David S. Miller <davem@davemloft.net>
|
|
---
|
|
net/ipv4/tcp_output.c | 13 ++++++++++---
|
|
1 file changed, 10 insertions(+), 3 deletions(-)
|
|
|
|
--- a/net/ipv4/tcp_output.c
|
|
+++ b/net/ipv4/tcp_output.c
|
|
@@ -860,6 +860,7 @@ void tcp_wfree(struct sk_buff *skb)
|
|
{
|
|
struct sock *sk = skb->sk;
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
|
+ unsigned long flags, nval, oval;
|
|
int wmem;
|
|
|
|
/* Keep one reference on sk_wmem_alloc.
|
|
@@ -877,11 +878,17 @@ void tcp_wfree(struct sk_buff *skb)
|
|
if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
|
|
goto out;
|
|
|
|
- if (test_and_clear_bit(TSQ_THROTTLED, &tp->tsq_flags) &&
|
|
- !test_and_set_bit(TSQ_QUEUED, &tp->tsq_flags)) {
|
|
- unsigned long flags;
|
|
+ for (oval = READ_ONCE(tp->tsq_flags);; oval = nval) {
|
|
struct tsq_tasklet *tsq;
|
|
|
|
+ if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
|
|
+ goto out;
|
|
+
|
|
+ nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
|
|
+ nval = cmpxchg(&tp->tsq_flags, oval, nval);
|
|
+ if (nval != oval)
|
|
+ continue;
|
|
+
|
|
/* queue this socket to tasklet queue */
|
|
local_irq_save(flags);
|
|
tsq = this_cpu_ptr(&tsq_tasklet);
|