2019-05-27 06:55:01 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
2005-08-10 03:10:42 +00:00
|
|
|
/*
|
|
|
|
|
* NET Generic infrastructure for INET connection oriented protocols.
|
|
|
|
|
*
|
|
|
|
|
* Definitions for inet_connection_sock
|
|
|
|
|
*
|
|
|
|
|
* Authors: Many people, see the TCP sources
|
|
|
|
|
*
|
|
|
|
|
* From code originally in TCP
|
|
|
|
|
*/
|
|
|
|
|
#ifndef _INET_CONNECTION_SOCK_H
|
|
|
|
|
#define _INET_CONNECTION_SOCK_H
|
|
|
|
|
|
2005-12-14 07:15:52 +00:00
|
|
|
#include <linux/compiler.h>
|
2005-08-10 03:11:08 +00:00
|
|
|
#include <linux/string.h>
|
2005-08-10 03:10:42 +00:00
|
|
|
#include <linux/timer.h>
|
2006-10-19 21:23:57 +00:00
|
|
|
#include <linux/poll.h>
|
2018-08-01 21:57:59 +00:00
|
|
|
#include <linux/kernel.h>
|
2020-07-23 06:09:07 +00:00
|
|
|
#include <linux/sockptr.h>
|
2005-12-27 04:43:12 +00:00
|
|
|
|
|
|
|
|
#include <net/inet_sock.h>
|
2005-08-10 03:10:42 +00:00
|
|
|
#include <net/request_sock.h>
|
|
|
|
|
|
2005-08-10 03:11:08 +00:00
|
|
|
/* Cancel timers, when they are not required. */
|
|
|
|
|
#undef INET_CSK_CLEAR_TIMERS
|
|
|
|
|
|
2005-08-10 03:10:42 +00:00
|
|
|
struct inet_bind_bucket;
|
net: Add a bhash2 table hashed by port and address
The current bind hashtable (bhash) is hashed by port only.
In the socket bind path, we have to check for bind conflicts by
traversing the specified port's inet_bind_bucket while holding the
hashbucket's spinlock (see inet_csk_get_port() and
inet_csk_bind_conflict()). In instances where there are tons of
sockets hashed to the same port at different addresses, the bind
conflict check is time-intensive and can cause softirq cpu lockups,
as well as stops new tcp connections since __inet_inherit_port()
also contests for the spinlock.
This patch adds a second bind table, bhash2, that hashes by
port and sk->sk_rcv_saddr (ipv4) and sk->sk_v6_rcv_saddr (ipv6).
Searching the bhash2 table leads to significantly faster conflict
resolution and less time holding the hashbucket spinlock.
Please note a few things:
* There can be the case where the a socket's address changes after it
has been bound. There are two cases where this happens:
1) The case where there is a bind() call on INADDR_ANY (ipv4) or
IPV6_ADDR_ANY (ipv6) and then a connect() call. The kernel will
assign the socket an address when it handles the connect()
2) In inet_sk_reselect_saddr(), which is called when rebuilding the
sk header and a few pre-conditions are met (eg rerouting fails).
In these two cases, we need to update the bhash2 table by removing the
entry for the old address, and add a new entry reflecting the updated
address.
* The bhash2 table must have its own lock, even though concurrent
accesses on the same port are protected by the bhash lock. Bhash2 must
have its own lock to protect against cases where sockets on different
ports hash to different bhash hashbuckets but to the same bhash2
hashbucket.
This brings up a few stipulations:
1) When acquiring both the bhash and the bhash2 lock, the bhash2 lock
will always be acquired after the bhash lock and released before the
bhash lock is released.
2) There are no nested bhash2 hashbucket locks. A bhash2 lock is always
acquired+released before another bhash2 lock is acquired+released.
* The bhash table cannot be superseded by the bhash2 table because for
bind requests on INADDR_ANY (ipv4) or IPV6_ADDR_ANY (ipv6), every socket
bound to that port must be checked for a potential conflict. The bhash
table is the only source of port->socket associations.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-08-22 18:10:21 +00:00
|
|
|
struct inet_bind2_bucket;
|
2005-08-10 07:03:31 +00:00
|
|
|
struct tcp_congestion_ops;
|
2005-08-10 03:10:42 +00:00
|
|
|
|
2005-12-14 07:15:52 +00:00
|
|
|
/*
|
|
|
|
|
* Pointers to address related TCP functions
|
|
|
|
|
* (i.e. things that depend on the address family)
|
|
|
|
|
*/
|
|
|
|
|
struct inet_connection_sock_af_ops {
|
2014-04-15 16:58:34 +00:00
|
|
|
int (*queue_xmit)(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
|
2010-04-11 02:15:55 +00:00
|
|
|
void (*send_check)(struct sock *sk, struct sk_buff *skb);
|
2005-12-14 07:15:52 +00:00
|
|
|
int (*rebuild_header)(struct sock *sk);
|
2012-08-06 05:09:33 +00:00
|
|
|
void (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb);
|
2005-12-14 07:15:52 +00:00
|
|
|
int (*conn_request)(struct sock *sk, struct sk_buff *skb);
|
2015-09-29 14:42:48 +00:00
|
|
|
struct sock *(*syn_recv_sock)(const struct sock *sk, struct sk_buff *skb,
|
2005-12-14 07:15:52 +00:00
|
|
|
struct request_sock *req,
|
2015-10-22 15:20:46 +00:00
|
|
|
struct dst_entry *dst,
|
|
|
|
|
struct request_sock *req_unhash,
|
|
|
|
|
bool *own_req);
|
[INET_CONNECTION_SOCK]: Pack struct inet_connection_sock_af_ops
We have a hole in:
[acme@newtoy net-2.6.20]$ pahole net/ipv6/tcp_ipv6.o inet_connection_sock_af_ops
/* /pub/scm/linux/kernel/git/acme/net-2.6.20/include/net/inet_connection_sock.h:38 */
struct inet_connection_sock_af_ops {
int (*queue_xmit)(); /* 0 4 */
void (*send_check)(); /* 4 4 */
int (*rebuild_header)(); /* 8 4 */
int (*conn_request)(); /* 12 4 */
struct sock * (*syn_recv_sock)(); /* 16 4 */
int (*remember_stamp)(); /* 20 4 */
__u16 net_header_len; /* 24 2 */
/* XXX 2 bytes hole, try to pack */
int (*setsockopt)(); /* 28 4 */
int (*getsockopt)(); /* 32 4 */
int (*compat_setsockopt)(); /* 36 4 */
int (*compat_getsockopt)(); /* 40 4 */
void (*addr2sockaddr)(); /* 44 4 */
int sockaddr_len; /* 48 4 */
}; /* size: 52, sum members: 50, holes: 1, sum holes: 2 */
But we don't need sockaddr_len to be an int:
[acme@newtoy net-2.6.20]$ find net -name "*.[ch]" | xargs grep '\.sockaddr_len.\+=' | sort -u
net/dccp/ipv4.c: .sockaddr_len = sizeof(struct sockaddr_in),
net/dccp/ipv6.c: .sockaddr_len = sizeof(struct sockaddr_in6),
net/ipv4/tcp_ipv4.c: .sockaddr_len = sizeof(struct sockaddr_in),
net/ipv6/tcp_ipv6.c: .sockaddr_len = sizeof(struct sockaddr_in6),
net/sctp/ipv6.c: .sockaddr_len = sizeof(struct sockaddr_in6),
net/sctp/protocol.c: .sockaddr_len = sizeof(struct sockaddr_in),
[acme@newtoy net-2.6.20]$ pahole --sizes net/ipv6/tcp_ipv6.o | grep sockaddr_in
struct sockaddr_in: 16 0
struct sockaddr_in6: 28 0
[acme@newtoy net-2.6.20]$
So I turned sockaddr_len a 'u16', and now:
[acme@newtoy net-2.6.20]$ pahole net/ipv6/tcp_ipv6.o inet_connection_sock_af_ops
/* /pub/scm/linux/kernel/git/acme/net-2.6.20/include/net/inet_connection_sock.h:38 */
struct inet_connection_sock_af_ops {
int (*queue_xmit)(); /* 0 4 */
void (*send_check)(); /* 4 4 */
int (*rebuild_header)(); /* 8 4 */
int (*conn_request)(); /* 12 4 */
struct sock * (*syn_recv_sock)(); /* 16 4 */
int (*remember_stamp)(); /* 20 4 */
u16 net_header_len; /* 24 2 */
u16 sockaddr_len; /* 26 2 */
int (*setsockopt)(); /* 28 4 */
int (*getsockopt)(); /* 32 4 */
int (*compat_setsockopt)(); /* 36 4 */
int (*compat_getsockopt)(); /* 40 4 */
void (*addr2sockaddr)(); /* 44 4 */
}; /* size: 48 */
So we've saved 4 bytes:
[acme@newtoy net-2.6.20]$ codiff -sV /tmp/tcp_ipv6.o.before net/ipv6/tcp_ipv6.o
/pub/scm/linux/kernel/git/acme/net-2.6.20/net/ipv6/tcp_ipv6.c:
struct inet_connection_sock_af_ops | -4
net_header_len;
from: __u16 /* 24(0) 2(0) */
to: u16 /* 24(0) 2(0) */
sockaddr_len;
from: int /* 48(0) 4(0) */
to: u16 /* 26(0) 2(0) */
1 struct changed
[acme@newtoy net-2.6.20]$
Signed-off-by: Arnaldo Carvalho de Melo <acme@mandriva.com>
2006-11-27 19:56:43 +00:00
|
|
|
u16 net_header_len;
|
|
|
|
|
u16 sockaddr_len;
|
2018-02-27 23:48:21 +00:00
|
|
|
int (*setsockopt)(struct sock *sk, int level, int optname,
|
2020-07-23 06:09:07 +00:00
|
|
|
sockptr_t optval, unsigned int optlen);
|
2018-02-27 23:48:21 +00:00
|
|
|
int (*getsockopt)(struct sock *sk, int level, int optname,
|
2005-12-14 07:15:52 +00:00
|
|
|
char __user *optval, int __user *optlen);
|
|
|
|
|
void (*addr2sockaddr)(struct sock *sk, struct sockaddr *);
|
2014-08-14 16:40:05 +00:00
|
|
|
void (*mtu_reduced)(struct sock *sk);
|
2005-12-14 07:15:52 +00:00
|
|
|
};
|
|
|
|
|
|
2005-08-10 03:10:42 +00:00
|
|
|
/** inet_connection_sock - INET connection oriented sock
|
|
|
|
|
*
|
2018-02-27 23:48:21 +00:00
|
|
|
* @icsk_accept_queue: FIFO of established children
|
2005-08-10 03:10:42 +00:00
|
|
|
* @icsk_bind_hash: Bind node
|
net: Add a bhash2 table hashed by port and address
The current bind hashtable (bhash) is hashed by port only.
In the socket bind path, we have to check for bind conflicts by
traversing the specified port's inet_bind_bucket while holding the
hashbucket's spinlock (see inet_csk_get_port() and
inet_csk_bind_conflict()). In instances where there are tons of
sockets hashed to the same port at different addresses, the bind
conflict check is time-intensive and can cause softirq cpu lockups,
as well as stops new tcp connections since __inet_inherit_port()
also contests for the spinlock.
This patch adds a second bind table, bhash2, that hashes by
port and sk->sk_rcv_saddr (ipv4) and sk->sk_v6_rcv_saddr (ipv6).
Searching the bhash2 table leads to significantly faster conflict
resolution and less time holding the hashbucket spinlock.
Please note a few things:
* There can be the case where the a socket's address changes after it
has been bound. There are two cases where this happens:
1) The case where there is a bind() call on INADDR_ANY (ipv4) or
IPV6_ADDR_ANY (ipv6) and then a connect() call. The kernel will
assign the socket an address when it handles the connect()
2) In inet_sk_reselect_saddr(), which is called when rebuilding the
sk header and a few pre-conditions are met (eg rerouting fails).
In these two cases, we need to update the bhash2 table by removing the
entry for the old address, and add a new entry reflecting the updated
address.
* The bhash2 table must have its own lock, even though concurrent
accesses on the same port are protected by the bhash lock. Bhash2 must
have its own lock to protect against cases where sockets on different
ports hash to different bhash hashbuckets but to the same bhash2
hashbucket.
This brings up a few stipulations:
1) When acquiring both the bhash and the bhash2 lock, the bhash2 lock
will always be acquired after the bhash lock and released before the
bhash lock is released.
2) There are no nested bhash2 hashbucket locks. A bhash2 lock is always
acquired+released before another bhash2 lock is acquired+released.
* The bhash table cannot be superseded by the bhash2 table because for
bind requests on INADDR_ANY (ipv4) or IPV6_ADDR_ANY (ipv6), every socket
bound to that port must be checked for a potential conflict. The bhash
table is the only source of port->socket associations.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-08-22 18:10:21 +00:00
|
|
|
* @icsk_bind2_hash: Bind node in the bhash2 table
|
2005-08-10 03:10:42 +00:00
|
|
|
* @icsk_timeout: Timeout
|
|
|
|
|
* @icsk_retransmit_timer: Resend (no ack)
|
|
|
|
|
* @icsk_rto: Retransmit timeout
|
2005-12-14 07:26:10 +00:00
|
|
|
* @icsk_pmtu_cookie Last pmtu seen by socket
|
2005-08-10 07:03:31 +00:00
|
|
|
* @icsk_ca_ops Pluggable congestion control hook
|
2005-12-14 07:15:52 +00:00
|
|
|
* @icsk_af_ops Operations which are AF_INET{4,6} specific
|
2017-06-14 18:37:14 +00:00
|
|
|
* @icsk_ulp_ops Pluggable ULP control hook
|
|
|
|
|
* @icsk_ulp_data ULP private data
|
2018-04-30 07:16:10 +00:00
|
|
|
* @icsk_clean_acked Clean acked data hook
|
2005-08-10 07:03:31 +00:00
|
|
|
* @icsk_ca_state: Congestion control state
|
2005-08-10 03:10:42 +00:00
|
|
|
* @icsk_retransmits: Number of unrecovered [RTO] timeouts
|
|
|
|
|
* @icsk_pending: Scheduled timer event
|
|
|
|
|
* @icsk_backoff: Backoff
|
|
|
|
|
* @icsk_syn_retries: Number of allowed SYN (or equivalent) retries
|
2005-08-10 07:03:31 +00:00
|
|
|
* @icsk_probes_out: unanswered 0 window probes
|
2005-12-14 07:26:10 +00:00
|
|
|
* @icsk_ext_hdr_len: Network protocol overhead (IP/IPv6 options)
|
2005-08-10 03:10:42 +00:00
|
|
|
* @icsk_ack: Delayed ACK control data
|
2006-03-21 01:53:41 +00:00
|
|
|
* @icsk_mtup; MTU probing control data
|
2021-01-15 22:30:58 +00:00
|
|
|
* @icsk_probes_tstamp: Probe timestamp (cleared by non-zero window ack)
|
|
|
|
|
* @icsk_user_timeout: TCP_USER_TIMEOUT value
|
2005-08-10 03:10:42 +00:00
|
|
|
*/
|
|
|
|
|
struct inet_connection_sock {
|
|
|
|
|
/* inet_sock has to be the first member! */
|
|
|
|
|
struct inet_sock icsk_inet;
|
|
|
|
|
struct request_sock_queue icsk_accept_queue;
|
|
|
|
|
struct inet_bind_bucket *icsk_bind_hash;
|
net: Add a bhash2 table hashed by port and address
The current bind hashtable (bhash) is hashed by port only.
In the socket bind path, we have to check for bind conflicts by
traversing the specified port's inet_bind_bucket while holding the
hashbucket's spinlock (see inet_csk_get_port() and
inet_csk_bind_conflict()). In instances where there are tons of
sockets hashed to the same port at different addresses, the bind
conflict check is time-intensive and can cause softirq cpu lockups,
as well as stops new tcp connections since __inet_inherit_port()
also contests for the spinlock.
This patch adds a second bind table, bhash2, that hashes by
port and sk->sk_rcv_saddr (ipv4) and sk->sk_v6_rcv_saddr (ipv6).
Searching the bhash2 table leads to significantly faster conflict
resolution and less time holding the hashbucket spinlock.
Please note a few things:
* There can be the case where the a socket's address changes after it
has been bound. There are two cases where this happens:
1) The case where there is a bind() call on INADDR_ANY (ipv4) or
IPV6_ADDR_ANY (ipv6) and then a connect() call. The kernel will
assign the socket an address when it handles the connect()
2) In inet_sk_reselect_saddr(), which is called when rebuilding the
sk header and a few pre-conditions are met (eg rerouting fails).
In these two cases, we need to update the bhash2 table by removing the
entry for the old address, and add a new entry reflecting the updated
address.
* The bhash2 table must have its own lock, even though concurrent
accesses on the same port are protected by the bhash lock. Bhash2 must
have its own lock to protect against cases where sockets on different
ports hash to different bhash hashbuckets but to the same bhash2
hashbucket.
This brings up a few stipulations:
1) When acquiring both the bhash and the bhash2 lock, the bhash2 lock
will always be acquired after the bhash lock and released before the
bhash lock is released.
2) There are no nested bhash2 hashbucket locks. A bhash2 lock is always
acquired+released before another bhash2 lock is acquired+released.
* The bhash table cannot be superseded by the bhash2 table because for
bind requests on INADDR_ANY (ipv4) or IPV6_ADDR_ANY (ipv6), every socket
bound to that port must be checked for a potential conflict. The bhash
table is the only source of port->socket associations.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-08-22 18:10:21 +00:00
|
|
|
struct inet_bind2_bucket *icsk_bind2_hash;
|
2005-08-10 03:10:42 +00:00
|
|
|
unsigned long icsk_timeout;
|
|
|
|
|
struct timer_list icsk_retransmit_timer;
|
|
|
|
|
struct timer_list icsk_delack_timer;
|
|
|
|
|
__u32 icsk_rto;
|
2020-08-20 19:00:27 +00:00
|
|
|
__u32 icsk_rto_min;
|
2020-08-20 19:00:21 +00:00
|
|
|
__u32 icsk_delack_max;
|
2005-12-14 07:26:10 +00:00
|
|
|
__u32 icsk_pmtu_cookie;
|
2006-01-09 23:56:09 +00:00
|
|
|
const struct tcp_congestion_ops *icsk_ca_ops;
|
|
|
|
|
const struct inet_connection_sock_af_ops *icsk_af_ops;
|
2017-06-14 18:37:14 +00:00
|
|
|
const struct tcp_ulp_ops *icsk_ulp_ops;
|
2019-08-30 10:25:47 +00:00
|
|
|
void __rcu *icsk_ulp_data;
|
2018-04-30 07:16:10 +00:00
|
|
|
void (*icsk_clean_acked)(struct sock *sk, u32 acked_seq);
|
2005-12-14 07:26:10 +00:00
|
|
|
unsigned int (*icsk_sync_mss)(struct sock *sk, u32 pmtu);
|
2020-09-10 19:35:32 +00:00
|
|
|
__u8 icsk_ca_state:5,
|
|
|
|
|
icsk_ca_initialized:1,
|
2015-05-29 17:47:07 +00:00
|
|
|
icsk_ca_setsockopt:1,
|
net: tcp: add key management to congestion control
This patch adds necessary infrastructure to the congestion control
framework for later per route congestion control support.
For a per route congestion control possibility, our aim is to store
a unique u32 key identifier into dst metrics, which can then be
mapped into a tcp_congestion_ops struct. We argue that having a
RTAX key entry is the most simple, generic and easy way to manage,
and also keeps the memory footprint of dst entries lower on 64 bit
than with storing a pointer directly, for example. Having a unique
key id also allows for decoupling actual TCP congestion control
module management from the FIB layer, i.e. we don't have to care
about expensive module refcounting inside the FIB at this point.
We first thought of using an IDR store for the realization, which
takes over dynamic assignment of unused key space and also performs
the key to pointer mapping in RCU. While doing so, we stumbled upon
the issue that due to the nature of dynamic key distribution, it
just so happens, arguably in very rare occasions, that excessive
module loads and unloads can lead to a possible reuse of previously
used key space. Thus, previously stale keys in the dst metric are
now being reassigned to a different congestion control algorithm,
which might lead to unexpected behaviour. One way to resolve this
would have been to walk FIBs on the actually rare occasion of a
module unload and reset the metric keys for each FIB in each netns,
but that's just very costly.
Therefore, we argue a better solution is to reuse the unique
congestion control algorithm name member and map that into u32 key
space through jhash. For that, we split the flags attribute (as it
currently uses 2 bits only anyway) into two u32 attributes, flags
and key, so that we can keep the cacheline boundary of 2 cachelines
on x86_64 and cache the precalculated key at registration time for
the fast path. On average we might expect 2 - 4 modules being loaded
worst case perhaps 15, so a key collision possibility is extremely
low, and guaranteed collision-free on LE/BE for all in-tree modules.
Overall this results in much simpler code, and all without the
overhead of an IDR. Due to the deterministic nature, modules can
now be unloaded, the congestion control algorithm for a specific
but unloaded key will fall back to the default one, and on module
reload time it will switch back to the expected algorithm
transparently.
Joint work with Florian Westphal.
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-01-05 22:57:46 +00:00
|
|
|
icsk_ca_dst_locked:1;
|
2005-08-10 03:10:42 +00:00
|
|
|
__u8 icsk_retransmits;
|
|
|
|
|
__u8 icsk_pending;
|
|
|
|
|
__u8 icsk_backoff;
|
|
|
|
|
__u8 icsk_syn_retries;
|
2005-08-10 07:03:31 +00:00
|
|
|
__u8 icsk_probes_out;
|
2005-12-14 07:26:10 +00:00
|
|
|
__u16 icsk_ext_hdr_len;
|
2005-08-10 03:10:42 +00:00
|
|
|
struct {
|
|
|
|
|
__u8 pending; /* ACK is pending */
|
|
|
|
|
__u8 quick; /* Scheduled number of quick acks */
|
|
|
|
|
__u8 pingpong; /* The session is interactive */
|
2020-09-30 12:54:57 +00:00
|
|
|
__u8 retry; /* Number of attempts */
|
2023-10-06 01:18:40 +00:00
|
|
|
#define ATO_BITS 8
|
|
|
|
|
__u32 ato:ATO_BITS, /* Predicted tick of soft clock */
|
|
|
|
|
lrcv_flowlabel:20, /* last received ipv6 flowlabel */
|
|
|
|
|
unused:4;
|
2005-08-10 03:10:42 +00:00
|
|
|
unsigned long timeout; /* Currently scheduled timeout */
|
|
|
|
|
__u32 lrcvtime; /* timestamp of last received data packet */
|
|
|
|
|
__u16 last_seg_size; /* Size of last incoming segment */
|
2018-02-27 23:48:21 +00:00
|
|
|
__u16 rcv_mss; /* MSS used for delayed ACK decisions */
|
2005-08-10 03:10:42 +00:00
|
|
|
} icsk_ack;
|
2006-03-21 01:53:41 +00:00
|
|
|
struct {
|
|
|
|
|
/* Range of MTUs to search */
|
|
|
|
|
int search_high;
|
|
|
|
|
int search_low;
|
|
|
|
|
|
|
|
|
|
/* Information on the current probe. */
|
2021-01-29 18:54:38 +00:00
|
|
|
u32 probe_size:31,
|
|
|
|
|
/* Is the MTUP feature enabled for this connection? */
|
|
|
|
|
enabled:1;
|
2015-03-06 03:18:24 +00:00
|
|
|
|
|
|
|
|
u32 probe_timestamp;
|
2006-03-21 01:53:41 +00:00
|
|
|
} icsk_mtup;
|
2021-01-15 22:30:58 +00:00
|
|
|
u32 icsk_probes_tstamp;
|
tcp: Add TCP_USER_TIMEOUT socket option.
This patch provides a "user timeout" support as described in RFC793. The
socket option is also needed for the the local half of RFC5482 "TCP User
Timeout Option".
TCP_USER_TIMEOUT is a TCP level socket option that takes an unsigned int,
when > 0, to specify the maximum amount of time in ms that transmitted
data may remain unacknowledged before TCP will forcefully close the
corresponding connection and return ETIMEDOUT to the application. If
0 is given, TCP will continue to use the system default.
Increasing the user timeouts allows a TCP connection to survive extended
periods without end-to-end connectivity. Decreasing the user timeouts
allows applications to "fail fast" if so desired. Otherwise it may take
upto 20 minutes with the current system defaults in a normal WAN
environment.
The socket option can be made during any state of a TCP connection, but
is only effective during the synchronized states of a connection
(ESTABLISHED, FIN-WAIT-1, FIN-WAIT-2, CLOSE-WAIT, CLOSING, or LAST-ACK).
Moreover, when used with the TCP keepalive (SO_KEEPALIVE) option,
TCP_USER_TIMEOUT will overtake keepalive to determine when to close a
connection due to keepalive failure.
The option does not change in anyway when TCP retransmits a packet, nor
when a keepalive probe will be sent.
This option, like many others, will be inherited by an acceptor from its
listener.
Signed-off-by: H.K. Jerry Chu <hkchu@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-08-27 19:13:28 +00:00
|
|
|
u32 icsk_user_timeout;
|
2015-05-19 00:06:14 +00:00
|
|
|
|
tcp_bbr: adapt cwnd based on ack aggregation estimation
Aggregation effects are extremely common with wifi, cellular, and cable
modem link technologies, ACK decimation in middleboxes, and LRO and GRO
in receiving hosts. The aggregation can happen in either direction,
data or ACKs, but in either case the aggregation effect is visible
to the sender in the ACK stream.
Previously BBR's sending was often limited by cwnd under severe ACK
aggregation/decimation because BBR sized the cwnd at 2*BDP. If packets
were acked in bursts after long delays (e.g. one ACK acking 5*BDP after
5*RTT), BBR's sending was halted after sending 2*BDP over 2*RTT, leaving
the bottleneck idle for potentially long periods. Note that loss-based
congestion control does not have this issue because when facing
aggregation it continues increasing cwnd after bursts of ACKs, growing
cwnd until the buffer is full.
To achieve good throughput in the presence of aggregation effects, this
algorithm allows the BBR sender to put extra data in flight to keep the
bottleneck utilized during silences in the ACK stream that it has evidence
to suggest were caused by aggregation.
A summary of the algorithm: when a burst of packets are acked by a
stretched ACK or a burst of ACKs or both, BBR first estimates the expected
amount of data that should have been acked, based on its estimated
bandwidth. Then the surplus ("extra_acked") is recorded in a windowed-max
filter to estimate the recent level of observed ACK aggregation. Then cwnd
is increased by the ACK aggregation estimate. The larger cwnd avoids BBR
being cwnd-limited in the face of ACK silences that recent history suggests
were caused by aggregation. As a sanity check, the ACK aggregation degree
is upper-bounded by the cwnd (at the time of measurement) and a global max
of BW * 100ms. The algorithm is further described by the following
presentation:
https://datatracker.ietf.org/meeting/101/materials/slides-101-iccrg-an-update-on-bbr-work-at-google-00
In our internal testing, we observed a significant increase in BBR
throughput (measured using netperf), in a basic wifi setup.
- Host1 (sender on ethernet) -> AP -> Host2 (receiver on wifi)
- 2.4 GHz -> BBR before: ~73 Mbps; BBR after: ~102 Mbps; CUBIC: ~100 Mbps
- 5.0 GHz -> BBR before: ~362 Mbps; BBR after: ~593 Mbps; CUBIC: ~601 Mbps
Also, this code is running globally on YouTube TCP connections and produced
significant bandwidth increases for YouTube traffic.
This is based on Ian Swett's max_ack_height_ algorithm from the
QUIC BBR implementation.
Signed-off-by: Priyaranjan Jha <priyarjha@google.com>
Signed-off-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-01-23 20:04:54 +00:00
|
|
|
u64 icsk_ca_priv[104 / sizeof(u64)];
|
2021-06-29 13:53:14 +00:00
|
|
|
#define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv)
|
2005-08-10 03:10:42 +00:00
|
|
|
};
|
|
|
|
|
|
2005-08-10 03:11:08 +00:00
|
|
|
#define ICSK_TIME_RETRANS 1 /* Retransmit timer */
|
|
|
|
|
#define ICSK_TIME_DACK 2 /* Delayed ack timer */
|
|
|
|
|
#define ICSK_TIME_PROBE0 3 /* Zero window probe timer */
|
tcp: Tail loss probe (TLP)
This patch series implement the Tail loss probe (TLP) algorithm described
in http://tools.ietf.org/html/draft-dukkipati-tcpm-tcp-loss-probe-01. The
first patch implements the basic algorithm.
TLP's goal is to reduce tail latency of short transactions. It achieves
this by converting retransmission timeouts (RTOs) occuring due
to tail losses (losses at end of transactions) into fast recovery.
TLP transmits one packet in two round-trips when a connection is in
Open state and isn't receiving any ACKs. The transmitted packet, aka
loss probe, can be either new or a retransmission. When there is tail
loss, the ACK from a loss probe triggers FACK/early-retransmit based
fast recovery, thus avoiding a costly RTO. In the absence of loss,
there is no change in the connection state.
PTO stands for probe timeout. It is a timer event indicating
that an ACK is overdue and triggers a loss probe packet. The PTO value
is set to max(2*SRTT, 10ms) and is adjusted to account for delayed
ACK timer when there is only one oustanding packet.
TLP Algorithm
On transmission of new data in Open state:
-> packets_out > 1: schedule PTO in max(2*SRTT, 10ms).
-> packets_out == 1: schedule PTO in max(2*RTT, 1.5*RTT + 200ms)
-> PTO = min(PTO, RTO)
Conditions for scheduling PTO:
-> Connection is in Open state.
-> Connection is either cwnd limited or no new data to send.
-> Number of probes per tail loss episode is limited to one.
-> Connection is SACK enabled.
When PTO fires:
new_segment_exists:
-> transmit new segment.
-> packets_out++. cwnd remains same.
no_new_packet:
-> retransmit the last segment.
Its ACK triggers FACK or early retransmit based recovery.
ACK path:
-> rearm RTO at start of ACK processing.
-> reschedule PTO if need be.
In addition, the patch includes a small variation to the Early Retransmit
(ER) algorithm, such that ER and TLP together can in principle recover any
N-degree of tail loss through fast recovery. TLP is controlled by the same
sysctl as ER, tcp_early_retrans sysctl.
tcp_early_retrans==0; disables TLP and ER.
==1; enables RFC5827 ER.
==2; delayed ER.
==3; TLP and delayed ER. [DEFAULT]
==4; TLP only.
The TLP patch series have been extensively tested on Google Web servers.
It is most effective for short Web trasactions, where it reduced RTOs by 15%
and improved HTTP response time (average by 6%, 99th percentile by 10%).
The transmitted probes account for <0.5% of the overall transmissions.
Signed-off-by: Nandita Dukkipati <nanditad@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-11 10:00:43 +00:00
|
|
|
#define ICSK_TIME_LOSS_PROBE 5 /* Tail loss probe timer */
|
2017-01-13 06:11:33 +00:00
|
|
|
#define ICSK_TIME_REO_TIMEOUT 6 /* Reordering timer */
|
2005-08-10 03:11:08 +00:00
|
|
|
|
2024-03-29 14:49:31 +00:00
|
|
|
#define inet_csk(ptr) container_of_const(ptr, struct inet_connection_sock, icsk_inet.sk)
|
2005-08-10 03:10:42 +00:00
|
|
|
|
2005-08-10 07:03:31 +00:00
|
|
|
static inline void *inet_csk_ca(const struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return (void *)inet_csk(sk)->icsk_ca_priv;
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
struct sock *inet_csk_clone_lock(const struct sock *sk,
|
|
|
|
|
const struct request_sock *req,
|
|
|
|
|
const gfp_t priority);
|
2005-08-10 03:11:24 +00:00
|
|
|
|
2005-08-10 03:11:08 +00:00
|
|
|
enum inet_csk_ack_state_t {
|
|
|
|
|
ICSK_ACK_SCHED = 1,
|
|
|
|
|
ICSK_ACK_TIMER = 2,
|
2006-09-19 19:52:50 +00:00
|
|
|
ICSK_ACK_PUSHED = 4,
|
2018-08-09 16:38:09 +00:00
|
|
|
ICSK_ACK_PUSHED2 = 8,
|
2023-08-11 02:55:27 +00:00
|
|
|
ICSK_ACK_NOW = 16, /* Send the next ACK immediately (once) */
|
|
|
|
|
ICSK_ACK_NOMEM = 32,
|
2005-08-10 03:11:08 +00:00
|
|
|
};
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_init_xmit_timers(struct sock *sk,
|
2017-10-17 00:29:19 +00:00
|
|
|
void (*retransmit_handler)(struct timer_list *),
|
|
|
|
|
void (*delack_handler)(struct timer_list *),
|
|
|
|
|
void (*keepalive_handler)(struct timer_list *));
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_clear_xmit_timers(struct sock *sk);
|
2024-03-22 13:57:32 +00:00
|
|
|
void inet_csk_clear_xmit_timers_sync(struct sock *sk);
|
2005-08-10 03:10:42 +00:00
|
|
|
|
2005-08-10 03:11:08 +00:00
|
|
|
static inline void inet_csk_schedule_ack(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_SCHED;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int inet_csk_ack_scheduled(const struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return inet_csk(sk)->icsk_ack.pending & ICSK_ACK_SCHED;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inet_csk_delack_init(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
memset(&inet_csk(sk)->icsk_ack, 0, sizeof(inet_csk(sk)->icsk_ack));
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_delete_keepalive_timer(struct sock *sk);
|
|
|
|
|
void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
|
2005-08-10 03:11:08 +00:00
|
|
|
|
|
|
|
|
static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)
|
|
|
|
|
{
|
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
2018-02-27 23:48:21 +00:00
|
|
|
|
2005-08-10 03:11:08 +00:00
|
|
|
if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0) {
|
|
|
|
|
icsk->icsk_pending = 0;
|
|
|
|
|
#ifdef INET_CSK_CLEAR_TIMERS
|
|
|
|
|
sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
|
|
|
|
|
#endif
|
|
|
|
|
} else if (what == ICSK_TIME_DACK) {
|
2020-09-30 12:54:56 +00:00
|
|
|
icsk->icsk_ack.pending = 0;
|
2020-09-30 12:54:57 +00:00
|
|
|
icsk->icsk_ack.retry = 0;
|
2005-08-10 03:11:08 +00:00
|
|
|
#ifdef INET_CSK_CLEAR_TIMERS
|
|
|
|
|
sk_stop_timer(sk, &icsk->icsk_delack_timer);
|
|
|
|
|
#endif
|
2018-05-10 06:24:07 +00:00
|
|
|
} else {
|
|
|
|
|
pr_debug("inet_csk BUG: unknown timer value\n");
|
2005-08-10 03:11:08 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reset the retransmission timer
|
|
|
|
|
*/
|
|
|
|
|
static inline void inet_csk_reset_xmit_timer(struct sock *sk, const int what,
|
|
|
|
|
unsigned long when,
|
|
|
|
|
const unsigned long max_when)
|
|
|
|
|
{
|
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
|
|
|
|
|
|
if (when > max_when) {
|
|
|
|
|
pr_debug("reset_xmit_timer: sk=%p %d when=0x%lx, caller=%p\n",
|
2018-08-01 21:57:59 +00:00
|
|
|
sk, what, when, (void *)_THIS_IP_);
|
2005-08-10 03:11:08 +00:00
|
|
|
when = max_when;
|
|
|
|
|
}
|
|
|
|
|
|
tcp: Tail loss probe (TLP)
This patch series implement the Tail loss probe (TLP) algorithm described
in http://tools.ietf.org/html/draft-dukkipati-tcpm-tcp-loss-probe-01. The
first patch implements the basic algorithm.
TLP's goal is to reduce tail latency of short transactions. It achieves
this by converting retransmission timeouts (RTOs) occuring due
to tail losses (losses at end of transactions) into fast recovery.
TLP transmits one packet in two round-trips when a connection is in
Open state and isn't receiving any ACKs. The transmitted packet, aka
loss probe, can be either new or a retransmission. When there is tail
loss, the ACK from a loss probe triggers FACK/early-retransmit based
fast recovery, thus avoiding a costly RTO. In the absence of loss,
there is no change in the connection state.
PTO stands for probe timeout. It is a timer event indicating
that an ACK is overdue and triggers a loss probe packet. The PTO value
is set to max(2*SRTT, 10ms) and is adjusted to account for delayed
ACK timer when there is only one oustanding packet.
TLP Algorithm
On transmission of new data in Open state:
-> packets_out > 1: schedule PTO in max(2*SRTT, 10ms).
-> packets_out == 1: schedule PTO in max(2*RTT, 1.5*RTT + 200ms)
-> PTO = min(PTO, RTO)
Conditions for scheduling PTO:
-> Connection is in Open state.
-> Connection is either cwnd limited or no new data to send.
-> Number of probes per tail loss episode is limited to one.
-> Connection is SACK enabled.
When PTO fires:
new_segment_exists:
-> transmit new segment.
-> packets_out++. cwnd remains same.
no_new_packet:
-> retransmit the last segment.
Its ACK triggers FACK or early retransmit based recovery.
ACK path:
-> rearm RTO at start of ACK processing.
-> reschedule PTO if need be.
In addition, the patch includes a small variation to the Early Retransmit
(ER) algorithm, such that ER and TLP together can in principle recover any
N-degree of tail loss through fast recovery. TLP is controlled by the same
sysctl as ER, tcp_early_retrans sysctl.
tcp_early_retrans==0; disables TLP and ER.
==1; enables RFC5827 ER.
==2; delayed ER.
==3; TLP and delayed ER. [DEFAULT]
==4; TLP only.
The TLP patch series have been extensively tested on Google Web servers.
It is most effective for short Web trasactions, where it reduced RTOs by 15%
and improved HTTP response time (average by 6%, 99th percentile by 10%).
The transmitted probes account for <0.5% of the overall transmissions.
Signed-off-by: Nandita Dukkipati <nanditad@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-11 10:00:43 +00:00
|
|
|
if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0 ||
|
2021-01-21 14:31:13 +00:00
|
|
|
what == ICSK_TIME_LOSS_PROBE || what == ICSK_TIME_REO_TIMEOUT) {
|
2005-08-10 03:11:08 +00:00
|
|
|
icsk->icsk_pending = what;
|
|
|
|
|
icsk->icsk_timeout = jiffies + when;
|
|
|
|
|
sk_reset_timer(sk, &icsk->icsk_retransmit_timer, icsk->icsk_timeout);
|
|
|
|
|
} else if (what == ICSK_TIME_DACK) {
|
|
|
|
|
icsk->icsk_ack.pending |= ICSK_ACK_TIMER;
|
|
|
|
|
icsk->icsk_ack.timeout = jiffies + when;
|
|
|
|
|
sk_reset_timer(sk, &icsk->icsk_delack_timer, icsk->icsk_ack.timeout);
|
2018-05-10 06:24:07 +00:00
|
|
|
} else {
|
|
|
|
|
pr_debug("inet_csk BUG: unknown timer value\n");
|
2005-08-10 03:11:08 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-22 20:19:44 +00:00
|
|
|
static inline unsigned long
|
|
|
|
|
inet_csk_rto_backoff(const struct inet_connection_sock *icsk,
|
|
|
|
|
unsigned long max_when)
|
|
|
|
|
{
|
|
|
|
|
u64 when = (u64)icsk->icsk_rto << icsk->icsk_backoff;
|
|
|
|
|
|
|
|
|
|
return (unsigned long)min_t(u64, when, max_when);
|
|
|
|
|
}
|
|
|
|
|
|
2024-05-09 15:20:08 +00:00
|
|
|
struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg);
|
2005-08-10 03:11:08 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
int inet_csk_get_port(struct sock *sk, unsigned short snum);
|
2005-08-10 03:10:42 +00:00
|
|
|
|
2015-09-25 14:39:11 +00:00
|
|
|
struct dst_entry *inet_csk_route_req(const struct sock *sk, struct flowi4 *fl4,
|
2013-09-21 17:22:41 +00:00
|
|
|
const struct request_sock *req);
|
2015-09-29 14:42:43 +00:00
|
|
|
struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
|
|
|
|
|
struct sock *newsk,
|
2012-07-17 21:02:46 +00:00
|
|
|
const struct request_sock *req);
|
2005-08-10 03:10:42 +00:00
|
|
|
|
2016-02-18 13:39:18 +00:00
|
|
|
struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
|
|
|
|
|
struct request_sock *req,
|
|
|
|
|
struct sock *child);
|
Fix race for duplicate reqsk on identical SYN
When bonding is configured in BOND_MODE_BROADCAST mode, if two identical
SYN packets are received at the same time and processed on different CPUs,
it can potentially create the same sk (sock) but two different reqsk
(request_sock) in tcp_conn_request().
These two different reqsk will respond with two SYNACK packets, and since
the generation of the seq (ISN) incorporates a timestamp, the final two
SYNACK packets will have different seq values.
The consequence is that when the Client receives and replies with an ACK
to the earlier SYNACK packet, we will reset(RST) it.
========================================================================
This behavior is consistently reproducible in my local setup,
which comprises:
| NETA1 ------ NETB1 |
PC_A --- bond --- | | --- bond --- PC_B
| NETA2 ------ NETB2 |
- PC_A is the Server and has two network cards, NETA1 and NETA2. I have
bonded these two cards using BOND_MODE_BROADCAST mode and configured
them to be handled by different CPU.
- PC_B is the Client, also equipped with two network cards, NETB1 and
NETB2, which are also bonded and configured in BOND_MODE_BROADCAST mode.
If the client attempts a TCP connection to the server, it might encounter
a failure. Capturing packets from the server side reveals:
10.10.10.10.45182 > localhost: Flags [S], seq 320236027,
10.10.10.10.45182 > localhost: Flags [S], seq 320236027,
localhost > 10.10.10.10.45182: Flags [S.], seq 2967855116,
localhost > 10.10.10.10.45182: Flags [S.], seq 2967855123, <==
10.10.10.10.45182 > localhost: Flags [.], ack 4294967290,
10.10.10.10.45182 > localhost: Flags [.], ack 4294967290,
localhost > 10.10.10.10.45182: Flags [R], seq 2967855117, <==
localhost > 10.10.10.10.45182: Flags [R], seq 2967855117,
Two SYNACKs with different seq numbers are sent by localhost,
resulting in an anomaly.
========================================================================
The attempted solution is as follows:
Add a return value to inet_csk_reqsk_queue_hash_add() to confirm if the
ehash insertion is successful (Up to now, the reason for unsuccessful
insertion is that a reqsk for the same connection has already been
inserted). If the insertion fails, release the reqsk.
Due to the refcnt, Kuniyuki suggests also adding a return value check
for the DCCP module; if ehash insertion fails, indicating a successful
insertion of the same connection, simply release the reqsk as well.
Simultaneously, In the reqsk_queue_hash_req(), the start of the
req->rsk_timer is adjusted to be after successful insertion.
Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2")
Signed-off-by: luoxuanqiang <luoxuanqiang@kylinos.cn>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20240621013929.1386815-1-luoxuanqiang@kylinos.cn
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2024-06-21 01:39:29 +00:00
|
|
|
bool inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
|
2013-09-21 17:22:41 +00:00
|
|
|
unsigned long timeout);
|
2015-10-22 15:20:46 +00:00
|
|
|
struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
|
|
|
|
|
struct request_sock *req,
|
|
|
|
|
bool own_req);
|
2005-08-10 03:11:08 +00:00
|
|
|
|
2015-10-02 18:43:31 +00:00
|
|
|
static inline void inet_csk_reqsk_queue_added(struct sock *sk)
|
2005-08-10 03:11:08 +00:00
|
|
|
{
|
inet: get rid of central tcp/dccp listener timer
One of the major issue for TCP is the SYNACK rtx handling,
done by inet_csk_reqsk_queue_prune(), fired by the keepalive
timer of a TCP_LISTEN socket.
This function runs for awful long times, with socket lock held,
meaning that other cpus needing this lock have to spin for hundred of ms.
SYNACK are sent in huge bursts, likely to cause severe drops anyway.
This model was OK 15 years ago when memory was very tight.
We now can afford to have a timer per request sock.
Timer invocations no longer need to lock the listener,
and can be run from all cpus in parallel.
With following patch increasing somaxconn width to 32 bits,
I tested a listener with more than 4 million active request sockets,
and a steady SYNFLOOD of ~200,000 SYN per second.
Host was sending ~830,000 SYNACK per second.
This is ~100 times more what we could achieve before this patch.
Later, we will get rid of the listener hash and use ehash instead.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-03-20 02:04:20 +00:00
|
|
|
reqsk_queue_added(&inet_csk(sk)->icsk_accept_queue);
|
2005-08-10 03:11:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int inet_csk_reqsk_queue_len(const struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return reqsk_queue_len(&inet_csk(sk)->icsk_accept_queue);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int inet_csk_reqsk_queue_is_full(const struct sock *sk)
|
|
|
|
|
{
|
2024-03-31 09:05:21 +00:00
|
|
|
return inet_csk_reqsk_queue_len(sk) >= READ_ONCE(sk->sk_max_ack_backlog);
|
2005-08-10 03:11:08 +00:00
|
|
|
}
|
|
|
|
|
|
2021-03-15 11:05:45 +00:00
|
|
|
bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req);
|
2015-10-14 18:16:27 +00:00
|
|
|
void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req);
|
2005-08-10 03:11:08 +00:00
|
|
|
|
2022-01-28 19:26:21 +00:00
|
|
|
static inline unsigned long
|
|
|
|
|
reqsk_timeout(struct request_sock *req, unsigned long max_timeout)
|
|
|
|
|
{
|
|
|
|
|
u64 timeout = (u64)req->timeout << req->num_timeout;
|
|
|
|
|
|
|
|
|
|
return (unsigned long)min_t(u64, timeout, max_timeout);
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-15 17:22:16 +00:00
|
|
|
static inline void inet_csk_prepare_for_destroy_sock(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
/* The below has to be done to allow calling inet_csk_destroy_sock */
|
|
|
|
|
sock_set_flag(sk, SOCK_DEAD);
|
2021-10-14 13:41:26 +00:00
|
|
|
this_cpu_inc(*sk->sk_prot->orphan_count);
|
2020-05-15 17:22:16 +00:00
|
|
|
}
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_destroy_sock(struct sock *sk);
|
|
|
|
|
void inet_csk_prepare_forced_close(struct sock *sk);
|
2005-08-24 04:52:58 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* LISTEN is a special case for poll..
|
|
|
|
|
*/
|
2017-07-03 04:01:49 +00:00
|
|
|
static inline __poll_t inet_csk_listen_poll(const struct sock *sk)
|
2005-08-24 04:52:58 +00:00
|
|
|
{
|
|
|
|
|
return !reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue) ?
|
2018-02-11 22:34:03 +00:00
|
|
|
(EPOLLIN | EPOLLRDNORM) : 0;
|
2005-08-24 04:52:58 +00:00
|
|
|
}
|
|
|
|
|
|
2021-11-22 10:16:21 +00:00
|
|
|
int inet_csk_listen_start(struct sock *sk);
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_listen_stop(struct sock *sk);
|
2005-08-10 03:11:56 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr);
|
2005-12-14 07:16:04 +00:00
|
|
|
|
2020-08-11 18:33:23 +00:00
|
|
|
/* update the fast reuse flag when adding a socket */
|
|
|
|
|
void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
|
|
|
|
|
struct sock *sk);
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu);
|
2019-01-25 18:53:19 +00:00
|
|
|
|
|
|
|
|
static inline void inet_csk_enter_pingpong_mode(struct sock *sk)
|
|
|
|
|
{
|
2023-10-11 20:30:44 +00:00
|
|
|
inet_csk(sk)->icsk_ack.pingpong =
|
|
|
|
|
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pingpong_thresh);
|
2019-01-25 18:53:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inet_csk_exit_pingpong_mode(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
inet_csk(sk)->icsk_ack.pingpong = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool inet_csk_in_pingpong_mode(struct sock *sk)
|
|
|
|
|
{
|
2023-10-11 20:30:44 +00:00
|
|
|
return inet_csk(sk)->icsk_ack.pingpong >=
|
|
|
|
|
READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pingpong_thresh);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inet_csk_inc_pingpong_cnt(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
|
|
|
|
|
|
if (icsk->icsk_ack.pingpong < U8_MAX)
|
|
|
|
|
icsk->icsk_ack.pingpong++;
|
2019-01-25 18:53:19 +00:00
|
|
|
}
|
2019-01-25 18:53:20 +00:00
|
|
|
|
2023-08-16 08:15:42 +00:00
|
|
|
static inline bool inet_csk_has_ulp(const struct sock *sk)
|
2020-03-09 11:12:32 +00:00
|
|
|
{
|
2023-08-16 08:15:42 +00:00
|
|
|
return inet_test_bit(IS_ICSK, sk) && !!inet_csk(sk)->icsk_ulp_ops;
|
2020-03-09 11:12:32 +00:00
|
|
|
}
|
|
|
|
|
|
2024-01-18 01:20:19 +00:00
|
|
|
static inline void inet_init_csk_locks(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
|
|
|
|
|
|
spin_lock_init(&icsk->icsk_accept_queue.rskq_lock);
|
|
|
|
|
spin_lock_init(&icsk->icsk_accept_queue.fastopenq.lock);
|
|
|
|
|
}
|
|
|
|
|
|
2005-08-10 03:10:42 +00:00
|
|
|
#endif /* _INET_CONNECTION_SOCK_H */
|