#include <sys/cdefs.h>

#include <sys/types.h>

#include <sys/param.h>

#include <sys/systm.h>

#include <sys/kernel.h>

#include <sys/sockio.h>

#include <sys/mbuf.h>

#include <sys/malloc.h>

#include <sys/module.h>

#include <sys/socket.h>

#include <sys/sysctl.h>

#include <sys/lock.h>

#include <sys/mutex.h>

#include <sys/taskqueue.h>

#include <sys/smp.h>

#include <sys/time.h>

#include <machine/smp.h>

#include <vm/uma.h>

#include <vm/vm.h>

#include <vm/pmap.h>

#include <net/ethernet.h>

#include <net/if.h>

#include <net/if_var.h>

#include <net/if_arp.h>

#include <net/if_dl.h>

#include <net/if_types.h>

#include <net/if_media.h>

#include <net/if_vlan_var.h>

#include <net/bpf.h>

#include <netinet/in_systm.h>

#include <netinet/in.h>

#include <netinet/ip.h>

#include <netinet/ip6.h>

#include <netinet6/ip6_var.h>

#include <netinet/udp.h>

#include <netinet/tcp.h>

#include <netinet/sctp.h>

#include <machine/bus.h>

#include <machine/resource.h>

#include <sys/bus.h>

#include <sys/rman.h>

#include <dev/pci/pcivar.h>

#include <dev/pci/pcireg.h>

#include "opt_inet.h"

#include "opt_inet6.h"

#include <sys/selinfo.h>

#include <net/netmap.h>

#include <dev/netmap/netmap_kern.h>

#include <net/netmap_virt.h>

#include <dev/netmap/netmap_mem2.h>

#include <dev/virtio/network/virtio_net.h>

#ifndef PTNET_CSB_ALLOC

#error "No support for on-device CSB"

#ifndef INET

#error "INET not defined, cannot support offloadings"

#if __FreeBSD_version >= 1100000

static uint64_t ptnet_get_counter(if_t, ift_counter);

typedef struct ifnet *if_t;

#define if_getsoftc(_ifp) (_ifp)->if_softc

#ifdef DEBUG

#define DBG(x) x

#else /* !DEBUG */

#define DBG(x)

#endif /* !DEBUG */

extern int ptnet_vnet_hdr; /* Tunable parameter */

struct ptnet_softc;

struct ptnet_queue_stats {

};

struct ptnet_queue {

};

#define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock)

#define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock)

#define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock)

struct ptnet_softc {

};

#define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock)

#define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock)

static int ptnet_probe(device_t);

static int ptnet_attach(device_t);

static int ptnet_detach(device_t);

static int ptnet_suspend(device_t);

static int ptnet_resume(device_t);

static int ptnet_shutdown(device_t);

static void ptnet_init(void *opaque);

static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);

static int ptnet_init_locked(struct ptnet_softc *sc);

static int ptnet_stop(struct ptnet_softc *sc);

static int ptnet_transmit(if_t ifp, struct mbuf *m);

static int ptnet_drain_transmit_queue(struct ptnet_queue *pq,

unsigned int budget,

bool may_resched);

static void ptnet_qflush(if_t ifp);

static void ptnet_tx_task(void *context, int pending);

static int ptnet_media_change(if_t ifp);

static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);

#ifdef PTNETMAP_STATS

static void ptnet_tick(void *opaque);

static int ptnet_irqs_init(struct ptnet_softc *sc);

static void ptnet_irqs_fini(struct ptnet_softc *sc);

static uint32_t ptnet_nm_ptctl(if_t ifp, uint32_t cmd);

static int ptnet_nm_config(struct netmap_adapter *na, unsigned *txr,

unsigned *txd, unsigned *rxr, unsigned *rxd);

static void ptnet_update_vnet_hdr(struct ptnet_softc *sc);

static int ptnet_nm_register(struct netmap_adapter *na, int onoff);

static int ptnet_nm_txsync(struct netmap_kring *kring, int flags);

static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags);

static void ptnet_tx_intr(void *opaque);

static void ptnet_rx_intr(void *opaque);

static unsigned ptnet_rx_discard(struct netmap_kring *kring,

unsigned int head);

static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,

bool may_resched);

static void ptnet_rx_task(void *context, int pending);

#ifdef DEVICE_POLLING

static poll_handler_t ptnet_poll;

static device_method_t ptnet_methods[] = {

DEVMETHOD(device_probe, ptnet_probe),

DEVMETHOD(device_attach, ptnet_attach),

DEVMETHOD(device_detach, ptnet_detach),

DEVMETHOD(device_suspend, ptnet_suspend),

DEVMETHOD(device_resume, ptnet_resume),

DEVMETHOD(device_shutdown, ptnet_shutdown),

DEVMETHOD_END

};

static driver_t ptnet_driver = {

"ptnet",

ptnet_methods,

sizeof(struct ptnet_softc)

};

static devclass_t ptnet_devclass;

DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,

NULL, NULL, SI_ORDER_MIDDLE + 2);

static int

ptnet_probe(device_t dev)

{

if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||

pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {

return (ENXIO);

}

device_set_desc(dev, "ptnet network adapter");

return (BUS_PROBE_DEFAULT);

}

static inline void ptnet_kick(struct ptnet_queue *pq)

{

#ifdef PTNETMAP_STATS

pq->stats.kicks ++;

#endif /* PTNETMAP_STATS */

bus_write_4(pq->sc->iomem, pq->kick, 0);

}

#define PTNET_BUF_RING_SIZE 4096

#define PTNET_RX_BUDGET 512

#define PTNET_RX_BATCH 1

#define PTNET_TX_BUDGET 512

#define PTNET_TX_BATCH 64

#define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)

#define PTNET_MAX_PKT_SIZE 65536

#define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP | CSUM_SCTP)

#define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6 |\

CSUM_SCTP_IPV6)

#define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\

PTNET_CSUM_OFFLOAD_IPV6)

static int

ptnet_attach(device_t dev)

{

uint32_t ptfeatures = 0;

unsigned int num_rx_rings, num_tx_rings;

struct netmap_adapter na_arg;

unsigned int nifp_offset;

struct ptnet_softc *sc;

if_t ifp;

uint32_t macreg;

int err, rid;

int i;

sc = device_get_softc(dev);

sc->dev = dev;

/* Setup PCI resources. */

pci_enable_busmaster(dev);

rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);

sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,

RF_ACTIVE);

if (sc->iomem == NULL) {

device_printf(dev, "Failed to map I/O BAR\n");

return (ENXIO);

}

/* Negotiate features with the hypervisor. */

if (ptnet_vnet_hdr) {

ptfeatures |= PTNETMAP_F_VNET_HDR;

}

bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */

ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */

sc->ptfeatures = ptfeatures;

/* Allocate CSB and carry out CSB allocation protocol (CSBBAH first,

sc->csb = malloc(sizeof(struct ptnet_csb), M_DEVBUF,

M_NOWAIT | M_ZERO);

if (sc->csb == NULL) {

device_printf(dev, "Failed to allocate CSB\n");

err = ENOMEM;

goto err_path;

}

{

/*

uint64_t paddr = vtophys(sc->csb);

bus_write_4(sc->iomem, PTNET_IO_CSBBAH,

(paddr >> 32) & 0xffffffff);

bus_write_4(sc->iomem, PTNET_IO_CSBBAL, paddr & 0xffffffff);

}

num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);

num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);

sc->num_rings = num_tx_rings + num_rx_rings;

sc->num_tx_rings = num_tx_rings;

/* Allocate and initialize per-queue data structures. */

sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,

M_DEVBUF, M_NOWAIT | M_ZERO);

if (sc->queues == NULL) {

err = ENOMEM;

goto err_path;

}

sc->rxqueues = sc->queues + num_tx_rings;

for (i = 0; i < sc->num_rings; i++) {

struct ptnet_queue *pq = sc->queues + i;

pq->sc = sc;

pq->kring_id = i;

pq->kick = PTNET_IO_KICK_BASE + 4 * i;

pq->ptring = sc->csb->rings + i;

snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",

device_get_nameunit(dev), i);

mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);

if (i >= num_tx_rings) {

/* RX queue: fix kring_id. */

pq->kring_id -= num_tx_rings;

} else {

/* TX queue: allocate buf_ring. */

pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,

M_DEVBUF, M_NOWAIT, &pq->lock);

if (pq->bufring == NULL) {

err = ENOMEM;

goto err_path;

}

}

}

sc->min_tx_space = 64; /* Safe initial value. */

err = ptnet_irqs_init(sc);

if (err) {

goto err_path;

}

/* Setup Ethernet interface. */

sc->ifp = ifp = if_alloc(IFT_ETHER);

if (ifp == NULL) {

device_printf(dev, "Failed to allocate ifnet\n");

err = ENOMEM;

goto err_path;

}

if_initname(ifp, device_get_name(dev), device_get_unit(dev));

ifp->if_baudrate = IF_Gbps(10);

ifp->if_softc = sc;

ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;

ifp->if_init = ptnet_init;

ifp->if_ioctl = ptnet_ioctl;

#if __FreeBSD_version >= 1100000

ifp->if_get_counter = ptnet_get_counter;

ifp->if_transmit = ptnet_transmit;

ifp->if_qflush = ptnet_qflush;

ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,

ptnet_media_status);

ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);

ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);

macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);

sc->hwaddr[0] = (macreg >> 8) & 0xff;

sc->hwaddr[1] = macreg & 0xff;

macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);

sc->hwaddr[2] = (macreg >> 24) & 0xff;

sc->hwaddr[3] = (macreg >> 16) & 0xff;

sc->hwaddr[4] = (macreg >> 8) & 0xff;

sc->hwaddr[5] = macreg & 0xff;

ether_ifattach(ifp, sc->hwaddr);

ifp->if_hdrlen = sizeof(struct ether_vlan_header);

ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;

if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {

/* Similarly to what the vtnet driver does, we can emulate

ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6

| IFCAP_VLAN_HWCSUM

| IFCAP_TSO | IFCAP_LRO

| IFCAP_VLAN_HWTSO

| IFCAP_VLAN_HWTAGGING;

}

ifp->if_capenable = ifp->if_capabilities;

#ifdef DEVICE_POLLING

/* Don't enable polling by default. */

ifp->if_capabilities |= IFCAP_POLLING;

snprintf(sc->lock_name, sizeof(sc->lock_name),

"%s", device_get_nameunit(dev));

mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);

callout_init_mtx(&sc->tick, &sc->lock, 0);

/* Prepare a netmap_adapter struct instance to do netmap_attach(). */

nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);

memset(&na_arg, 0, sizeof(na_arg));

na_arg.ifp = ifp;

na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);

na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);

na_arg.num_tx_rings = num_tx_rings;

na_arg.num_rx_rings = num_rx_rings;

na_arg.nm_config = ptnet_nm_config;

na_arg.nm_krings_create = ptnet_nm_krings_create;

na_arg.nm_krings_delete = ptnet_nm_krings_delete;

na_arg.nm_dtor = ptnet_nm_dtor;

na_arg.nm_register = ptnet_nm_register;

na_arg.nm_txsync = ptnet_nm_txsync;

na_arg.nm_rxsync = ptnet_nm_rxsync;

netmap_pt_guest_attach(&na_arg, sc->csb, nifp_offset,

bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));

/* Now a netmap adapter for this ifp has been allocated, and it

sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);

/* If virtio-net header was negotiated, set the virt_hdr_len field in

ptnet_update_vnet_hdr(sc);

device_printf(dev, "%s() completed\n", __func__);

return (0);

err_path:

ptnet_detach(dev);

return err;

}

static int

ptnet_detach(device_t dev)

{

struct ptnet_softc *sc = device_get_softc(dev);

int i;

#ifdef DEVICE_POLLING

if (sc->ifp->if_capenable & IFCAP_POLLING) {

ether_poll_deregister(sc->ifp);

}

callout_drain(&sc->tick);

if (sc->queues) {

/* Drain taskqueues before calling if_detach. */

for (i = 0; i < sc->num_rings; i++) {

struct ptnet_queue *pq = sc->queues + i;

if (pq->taskq) {

taskqueue_drain(pq->taskq, &pq->task);

}

}

}

if (sc->ifp) {

ether_ifdetach(sc->ifp);

/* Uninitialize netmap adapters for this device. */

netmap_detach(sc->ifp);

ifmedia_removeall(&sc->media);

if_free(sc->ifp);

sc->ifp = NULL;

}

ptnet_irqs_fini(sc);

if (sc->csb) {

bus_write_4(sc->iomem, PTNET_IO_CSBBAH, 0);

bus_write_4(sc->iomem, PTNET_IO_CSBBAL, 0);

free(sc->csb, M_DEVBUF);

sc->csb = NULL;

}

if (sc->queues) {

for (i = 0; i < sc->num_rings; i++) {

struct ptnet_queue *pq = sc->queues + i;

if (mtx_initialized(&pq->lock)) {

mtx_destroy(&pq->lock);

}

if (pq->bufring != NULL) {

buf_ring_free(pq->bufring, M_DEVBUF);

}

}

free(sc->queues, M_DEVBUF);

sc->queues = NULL;

}

if (sc->iomem) {

bus_release_resource(dev, SYS_RES_IOPORT,

PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);

sc->iomem = NULL;

}

mtx_destroy(&sc->lock);

device_printf(dev, "%s() completed\n", __func__);

return (0);

}

static int

ptnet_suspend(device_t dev)

{

struct ptnet_softc *sc;

sc = device_get_softc(dev);

(void)sc;

return (0);

}

static int

ptnet_resume(device_t dev)

{

struct ptnet_softc *sc;

sc = device_get_softc(dev);

(void)sc;

return (0);

}

static int

ptnet_shutdown(device_t dev)

{

/*

return (ptnet_suspend(dev));

}

static int

ptnet_irqs_init(struct ptnet_softc *sc)

{

int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);

int nvecs = sc->num_rings;

device_t dev = sc->dev;

int err = ENOSPC;

int cpu_cur;

int i;

if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) {

device_printf(dev, "Could not find MSI-X capability\n");

return (ENXIO);

}

sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,

&rid, RF_ACTIVE);

if (sc->msix_mem == NULL) {

device_printf(dev, "Failed to allocate MSIX PCI BAR\n");

return (ENXIO);

}

if (pci_msix_count(dev) < nvecs) {

device_printf(dev, "Not enough MSI-X vectors\n");

goto err_path;

}

err = pci_alloc_msix(dev, &nvecs);

if (err) {

device_printf(dev, "Failed to allocate MSI-X vectors\n");

goto err_path;

}

for (i = 0; i < nvecs; i++) {

struct ptnet_queue *pq = sc->queues + i;

rid = i + 1;

pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,

RF_ACTIVE);

if (pq->irq == NULL) {

device_printf(dev, "Failed to allocate interrupt "

"for queue #%d\n", i);

err = ENOSPC;

goto err_path;

}

}

cpu_cur = CPU_FIRST();

for (i = 0; i < nvecs; i++) {

struct ptnet_queue *pq = sc->queues + i;

void (*handler)(void *) = ptnet_tx_intr;

if (i >= sc->num_tx_rings) {

handler = ptnet_rx_intr;

}

err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,

NULL /* intr_filter */, handler,

pq, &pq->cookie);

if (err) {

device_printf(dev, "Failed to register intr handler "

"for queue #%d\n", i);

goto err_path;

}

bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);

#if 0

bus_bind_intr(sc->dev, pq->irq, cpu_cur);

cpu_cur = CPU_NEXT(cpu_cur);

}

device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);

cpu_cur = CPU_FIRST();

for (i = 0; i < nvecs; i++) {

struct ptnet_queue *pq = sc->queues + i;

static void (*handler)(void *context, int pending);

handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task;

TASK_INIT(&pq->task, 0, handler, pq);

pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,

taskqueue_thread_enqueue, &pq->taskq);

taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",

device_get_nameunit(sc->dev), cpu_cur);

cpu_cur = CPU_NEXT(cpu_cur);

}

return 0;

err_path:

ptnet_irqs_fini(sc);

return err;

}

static void

ptnet_irqs_fini(struct ptnet_softc *sc)

{

device_t dev = sc->dev;

int i;

for (i = 0; i < sc->num_rings; i++) {

struct ptnet_queue *pq = sc->queues + i;

if (pq->taskq) {

taskqueue_free(pq->taskq);

pq->taskq = NULL;

}

if (pq->cookie) {

bus_teardown_intr(dev, pq->irq, pq->cookie);

pq->cookie = NULL;

}

if (pq->irq) {

bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);

pq->irq = NULL;

}

}

if (sc->msix_mem) {

pci_release_msi(dev);

bus_release_resource(dev, SYS_RES_MEMORY,

PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),

sc->msix_mem);

sc->msix_mem = NULL;

}

}

static void

ptnet_init(void *opaque)

{

struct ptnet_softc *sc = opaque;

PTNET_CORE_LOCK(sc);

ptnet_init_locked(sc);

PTNET_CORE_UNLOCK(sc);

}

static int

ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)

{

struct ptnet_softc *sc = if_getsoftc(ifp);

device_t dev = sc->dev;

struct ifreq *ifr = (struct ifreq *)data;

int mask, err = 0;

switch (cmd) {

case SIOCSIFFLAGS:

device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);

PTNET_CORE_LOCK(sc);

if (ifp->if_flags & IFF_UP) {

/* Network stack wants the iff to be up. */

err = ptnet_init_locked(sc);

} else {

/* Network stack wants the iff to be down. */

err = ptnet_stop(sc);

}

/* We don't need to do nothing to support IFF_PROMISC,

PTNET_CORE_UNLOCK(sc);

break;

case SIOCSIFCAP:

device_printf(dev, "SIOCSIFCAP %x %x\n",

ifr->ifr_reqcap, ifp->if_capenable);

mask = ifr->ifr_reqcap ^ ifp->if_capenable;

#ifdef DEVICE_POLLING

if (mask & IFCAP_POLLING) {

struct ptnet_queue *pq;

int i;

if (ifr->ifr_reqcap & IFCAP_POLLING) {

err = ether_poll_register(ptnet_poll, ifp);

if (err) {

break;

}

/* Stop queues and sync with taskqueues. */

ifp->if_drv_flags &= ~IFF_DRV_RUNNING;

for (i = 0; i < sc->num_rings; i++) {

pq = sc-> queues + i;

/* Make sure the worker sees the

PTNET_Q_LOCK(pq);

pq->ptring->guest_need_kick = 0;

PTNET_Q_UNLOCK(pq);

/* Wait for rescheduling to finish. */

if (pq->taskq) {

taskqueue_drain(pq->taskq,

&pq->task);

}

}

ifp->if_drv_flags |= IFF_DRV_RUNNING;

} else {

err = ether_poll_deregister(ifp);

for (i = 0; i < sc->num_rings; i++) {

pq = sc-> queues + i;

PTNET_Q_LOCK(pq);

pq->ptring->guest_need_kick = 1;

PTNET_Q_UNLOCK(pq);

}

}

}

#endif /* DEVICE_POLLING */

ifp->if_capenable = ifr->ifr_reqcap;

break;

case SIOCSIFMTU:

/* We support any reasonable MTU. */

if (ifr->ifr_mtu < ETHERMIN ||

ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {

err = EINVAL;

} else {

PTNET_CORE_LOCK(sc);

ifp->if_mtu = ifr->ifr_mtu;

PTNET_CORE_UNLOCK(sc);

}

break;

case SIOCSIFMEDIA:

case SIOCGIFMEDIA:

err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);

break;

default:

err = ether_ioctl(ifp, cmd, data);

break;

}

return err;

}

static int

ptnet_init_locked(struct ptnet_softc *sc)

{

if_t ifp = sc->ifp;

struct netmap_adapter *na_dr = &sc->ptna->dr.up;

struct netmap_adapter *na_nm = &sc->ptna->hwup.up;

unsigned int nm_buf_size;

int ret;

if (ifp->if_drv_flags & IFF_DRV_RUNNING) {

return 0; /* nothing to do */

}

device_printf(sc->dev, "%s\n", __func__);

/* Translate offload capabilities according to if_capenable. */

ifp->if_hwassist = 0;

if (ifp->if_capenable & IFCAP_TXCSUM)

ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;

if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)

ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;

if (ifp->if_capenable & IFCAP_TSO4)

ifp->if_hwassist |= CSUM_IP_TSO;

if (ifp->if_capenable & IFCAP_TSO6)

ifp->if_hwassist |= CSUM_IP6_TSO;

/*

netmap_update_config(na_dr);

ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);

if (ret) {

device_printf(sc->dev, "netmap_mem_finalize() failed\n");

return ret;

}

if (sc->ptna->backend_regifs == 0) {

ret = ptnet_nm_krings_create(na_nm);

if (ret) {

device_printf(sc->dev, "ptnet_nm_krings_create() "

"failed\n");

goto err_mem_finalize;

}

ret = netmap_mem_rings_create(na_dr);

if (ret) {

device_printf(sc->dev, "netmap_mem_rings_create() "

"failed\n");

goto err_rings_create;

}

ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);

if (ret) {

device_printf(sc->dev, "netmap_mem_get_lut() "

"failed\n");

goto err_get_lut;

}

}

ret = ptnet_nm_register(na_dr, 1 /* on */);

if (ret) {

goto err_register;

}

nm_buf_size = NETMAP_BUF_SIZE(na_dr);

KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));

sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;

device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,

sc->min_tx_space);

#ifdef PTNETMAP_STATS

callout_reset(&sc->tick, hz, ptnet_tick, sc);

ifp->if_drv_flags |= IFF_DRV_RUNNING;

return 0;

err_register:

memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));

err_get_lut:

netmap_mem_rings_delete(na_dr);

err_rings_create:

ptnet_nm_krings_delete(na_nm);

err_mem_finalize:

netmap_mem_deref(na_dr->nm_mem, na_dr);

return ret;

}

static int

ptnet_stop(struct ptnet_softc *sc)

{

if_t ifp = sc->ifp;

struct netmap_adapter *na_dr = &sc->ptna->dr.up;

struct netmap_adapter *na_nm = &sc->ptna->hwup.up;

int i;

device_printf(sc->dev, "%s\n", __func__);

if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {

return 0; /* nothing to do */

}

/* Clear the driver-ready flag, and synchronize with all the queues,

ifp->if_drv_flags &= ~IFF_DRV_RUNNING;

callout_stop(&sc->tick);

for (i = 0; i < sc->num_rings; i++) {

PTNET_Q_LOCK(sc->queues + i);

PTNET_Q_UNLOCK(sc->queues + i);

}

ptnet_nm_register(na_dr, 0 /* off */);

if (sc->ptna->backend_regifs == 0) {

netmap_mem_rings_delete(na_dr);

ptnet_nm_krings_delete(na_nm);

}

netmap_mem_deref(na_dr->nm_mem, na_dr);

return 0;

}

static void

ptnet_qflush(if_t ifp)

{

struct ptnet_softc *sc = if_getsoftc(ifp);

int i;

/* Flush all the bufrings and do the interface flush. */

for (i = 0; i < sc->num_rings; i++) {

struct ptnet_queue *pq = sc->queues + i;

struct mbuf *m;

PTNET_Q_LOCK(pq);

if (pq->bufring) {

while ((m = buf_ring_dequeue_sc(pq->bufring))) {

m_freem(m);

}

}

PTNET_Q_UNLOCK(pq);

}

if_qflush(ifp);

}

static int

ptnet_media_change(if_t ifp)

{

struct ptnet_softc *sc = if_getsoftc(ifp);

struct ifmedia *ifm = &sc->media;

if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {

return EINVAL;

}

return 0;

}

#if __FreeBSD_version >= 1100000

static uint64_t

ptnet_get_counter(if_t ifp, ift_counter cnt)

{

struct ptnet_softc *sc = if_getsoftc(ifp);

struct ptnet_queue_stats stats[2];

int i;

/* Accumulate statistics over the queues. */

memset(stats, 0, sizeof(stats));