37#include <sys/eventhandler.h>
38#include <sys/kernel.h>
40#include <sys/malloc.h>
42#include <sys/module.h>
44#include <sys/sockopt.h>
45#include <sys/sysctl.h>
46#include <sys/socket.h>
48#include <net/ethernet.h>
50#include <net/if_var.h>
51#include <net/if_types.h>
52#include <net/if_vlan_var.h>
53#include <net/if_llatbl.h>
60#include <netinet6/in6_var.h>
61#include <netinet6/in6_pcb.h>
62#include <netinet6/nd6.h>
74static eventhandler_tag listen_start_eh;
75static eventhandler_tag listen_stop_eh;
76static eventhandler_tag lle_event_eh;
79toedev_connect(
struct toedev *tod __unused,
struct socket *so __unused,
80 struct nhop_object *nh __unused,
struct sockaddr *nam __unused)
132 struct sockaddr *sa __unused,
uint8_t *lladdr __unused,
141 struct nhop_object *nh0 __unused,
struct nhop_object *nh1 __unused)
172 struct socket *so __unused)
180 int sopt_dir __unused,
int sopt_name __unused)
188 struct tcp_info *ti __unused)
196 struct ktls_session *tls __unused,
int direction __unused)
204 tcp_seq seq __unused,
int mtu __unused)
221 (
"%s: inp is not a TCP inp", __func__));
232 TAILQ_FOREACH(
tod, &toedev_list, link) {
233 if (t == NULL || t ==
tod)
246 (
"%s: t_state %s", __func__, tcpstates[tp->
t_state]));
261 (
"%s: t_state %s", __func__, tcpstates[tp->
t_state]));
264 TAILQ_FOREACH(tod, &toedev_list, link)
314 TAILQ_FOREACH(t, &toedev_list, link) {
321 TAILQ_INSERT_TAIL(&toedev_list, tod, link);
342 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) {
344 TAILQ_REMOVE(&toedev_list, tod, link);
358 struct inpcb *inp,
void *tod,
void *todctx,
uint8_t iptos)
364 todctx, iptos, htons(0));
369 struct tcphdr *th,
struct socket **lsop)
391 inp = in6_pcblookup(&
V_tcbinfo, &inc->inc6_faddr,
392 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport,
422 struct sockaddr_in6 sin6;
424 LLE_WLOCK_ASSERT(lle);
426 ifp = lltable_get_ifp(lle->lle_tbl);
427 family = lltable_get_af(lle->lle_tbl);
429 if (family != AF_INET && family != AF_INET6)
434 if ((family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) ||
435 (family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6)))
442 sa = (
struct sockaddr *)&sin6;
443 lltable_fill_sa_entry(lle, sa);
447 if (evt != LLENTRY_RESOLVED) {
455 KASSERT(lle->la_flags & LLE_VALID,
456 (
"%s: %p resolved but not valid?", __func__, lle));
458 lladdr = (
uint8_t *)lle->ll_addr;
463 tod->
tod_l2_update(tod, ifp, sa, lladdr, EVL_MAKETAG(vid, pcp, 0));
478 switch (sa->sa_family) {
481 rc =
arpresolve(ifp, 0, NULL, sa, lladdr, NULL, NULL);
486 rc = nd6_resolve(ifp, LLE_SF(AF_INET6, 0), NULL, sa, lladdr,
491 return (EPROTONOSUPPORT);
497 if (ifp->if_type == IFT_L2VLAN) {
500 }
else if (ifp->if_pcp != IFNET_PCP_NONE) {
504 *vtag = EVL_MAKETAG(vid, pcp, 0);
521 (
"%s: tp %p not offloaded.", __func__, tp));
533 (
"%s: tp %p still offloaded.", __func__, tp));
535 if (tcp_output(tp) < 0)
550 mtx_init(&
toedev_lock,
"toedev lock", NULL, MTX_DEF);
551 TAILQ_INIT(&toedev_list);
557 lle_event_eh = EVENTHANDLER_REGISTER(lle_event,
toe_lle_event, NULL,
558 EVENTHANDLER_PRI_ANY);
568 if (!TAILQ_EMPTY(&toedev_list)) {
575 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh);
590 if (cmd == MOD_UNLOAD)
int arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, const struct sockaddr *dst, u_char *desten, uint32_t *pflags, struct llentry **plle)
void inp_apply_all(void(*func)(struct inpcb *, void *), void *arg)
struct inpcb * in_pcblookup(struct inpcbinfo *, struct in_addr, u_int, struct in_addr, u_int, int, struct ifnet *)
#define INP_WLOCK_ASSERT(inp)
#define INP_RLOCK_ASSERT(inp)
struct socket * inp_socket
struct inpcbinfo * inp_pcbinfo
int(* tod_syncache_respond)(struct toedev *, void *, struct mbuf *)
int(* tod_send_fin)(struct toedev *, struct tcpcb *)
int(* tod_listen_stop)(struct toedev *, struct tcpcb *)
void(* tod_pmtu_update)(struct toedev *, struct tcpcb *, tcp_seq, int)
void(* tod_offload_socket)(struct toedev *, void *, struct socket *)
int(* tod_connect)(struct toedev *, struct socket *, struct nhop_object *, struct sockaddr *)
void(* tod_pcb_detach)(struct toedev *, struct tcpcb *)
void(* tod_tcp_info)(struct toedev *, struct tcpcb *, struct tcp_info *)
int(* tod_send_rst)(struct toedev *, struct tcpcb *)
void(* tod_input)(struct toedev *, struct tcpcb *, struct mbuf *)
int(* tod_output)(struct toedev *, struct tcpcb *)
void(* tod_syncache_removed)(struct toedev *, void *)
int(* tod_alloc_tls_session)(struct toedev *, struct tcpcb *, struct ktls_session *, int)
void(* tod_l2_update)(struct toedev *, struct ifnet *, struct sockaddr *, uint8_t *, uint16_t)
int(* tod_listen_start)(struct toedev *, struct tcpcb *)
void(* tod_syncache_added)(struct toedev *, void *)
void(* tod_rcvd)(struct toedev *, struct tcpcb *)
void(* tod_route_redirect)(struct toedev *, struct ifnet *, struct nhop_object *, struct nhop_object *)
void(* tod_ctloutput)(struct toedev *, struct tcpcb *, int, int)
void tcp_offload_listen_stop(struct tcpcb *tp)
void tcp_offload_listen_start(struct tcpcb *tp)
struct tcpcb * tcp_drop(struct tcpcb *tp, int errno)
struct socket * syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, struct inpcb *inp, struct socket *so, struct mbuf *m, void *tod, void *todctx, uint8_t iptos, uint16_t port)
int syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, struct socket **lsop, struct mbuf *m, uint16_t port)
void tcp_timer_activate(struct tcpcb *tp, uint32_t timer_type, u_int delta)
int tcp_twcheck(struct inpcb *inp, struct tcpopt *to, struct tcphdr *th, struct mbuf *m, int tlen)
static int toecore_load(void)
static moduledata_t mod_data
int register_toedev(struct toedev *tod)
static void toe_listen_start_event(void *arg __unused, struct tcpcb *tp)
static void toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, struct nhop_object *nh0 __unused, struct nhop_object *nh1 __unused)
static void toe_listen_stop_event(void *arg __unused, struct tcpcb *tp)
static int toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused)
static int toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, struct mbuf *m)
static int toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused)
static void toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, struct socket *so __unused)
static void toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, struct sockaddr *sa __unused, uint8_t *lladdr __unused, uint16_t vtag __unused)
static void toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused, struct tcp_info *ti __unused)
void toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err)
static int toecore_unload(void)
int toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, struct socket **lsop)
static void toedev_pmtu_update(struct toedev *tod __unused, struct tcpcb *tp __unused, tcp_seq seq __unused, int mtu __unused)
static void toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused)
static void toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, int sopt_dir __unused, int sopt_name __unused)
static int toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, struct ktls_session *tls __unused, int direction __unused)
static TAILQ_HEAD(toedev)
int toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr, uint16_t *vtag)
static void toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused)
static void toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused)
static void toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, struct mbuf *m)
MODULE_VERSION(toecore, 1)
static void toe_lle_event(void *arg __unused, struct llentry *lle, int evt)
void init_toedev(struct toedev *tod)
static int toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused)
int toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp)
DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY)
static int toecore_mod_handler(module_t mod, int cmd, void *arg)
int unregister_toedev(struct toedev *tod)
void toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, struct inpcb *inp, void *tod, void *todctx, uint8_t iptos)
static void toe_listen_start(struct inpcb *inp, void *arg)
static void toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused)
static struct mtx toedev_lock