diff --git a/sys/dev/pci/if_vioif.c b/sys/dev/pci/if_vioif.c index 3bbd300e88e..769b108e793 100644 --- a/sys/dev/pci/if_vioif.c +++ b/sys/dev/pci/if_vioif.c @@ -45,6 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.41 2018/06/26 06:48:01 msaitoh Exp $" #include #include #include +#include #include #include @@ -71,28 +72,35 @@ __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.41 2018/06/26 06:48:01 msaitoh Exp $" /* Configuration registers */ #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ +#define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ /* Feature bits */ -#define VIRTIO_NET_F_CSUM (1<<0) -#define VIRTIO_NET_F_GUEST_CSUM (1<<1) -#define VIRTIO_NET_F_MAC (1<<5) -#define VIRTIO_NET_F_GSO (1<<6) -#define VIRTIO_NET_F_GUEST_TSO4 (1<<7) -#define VIRTIO_NET_F_GUEST_TSO6 (1<<8) -#define VIRTIO_NET_F_GUEST_ECN (1<<9) -#define VIRTIO_NET_F_GUEST_UFO (1<<10) -#define VIRTIO_NET_F_HOST_TSO4 (1<<11) -#define VIRTIO_NET_F_HOST_TSO6 (1<<12) -#define VIRTIO_NET_F_HOST_ECN (1<<13) -#define VIRTIO_NET_F_HOST_UFO (1<<14) -#define VIRTIO_NET_F_MRG_RXBUF (1<<15) -#define VIRTIO_NET_F_STATUS (1<<16) -#define VIRTIO_NET_F_CTRL_VQ (1<<17) -#define VIRTIO_NET_F_CTRL_RX (1<<18) -#define VIRTIO_NET_F_CTRL_VLAN (1<<19) +#define VIRTIO_NET_F_CSUM (1<<0) +#define VIRTIO_NET_F_GUEST_CSUM (1<<1) +#define VIRTIO_NET_F_MAC (1<<5) +#define VIRTIO_NET_F_GSO (1<<6) +#define VIRTIO_NET_F_GUEST_TSO4 (1<<7) +#define VIRTIO_NET_F_GUEST_TSO6 (1<<8) +#define VIRTIO_NET_F_GUEST_ECN (1<<9) +#define VIRTIO_NET_F_GUEST_UFO (1<<10) +#define VIRTIO_NET_F_HOST_TSO4 (1<<11) +#define VIRTIO_NET_F_HOST_TSO6 (1<<12) +#define VIRTIO_NET_F_HOST_ECN (1<<13) +#define VIRTIO_NET_F_HOST_UFO (1<<14) +#define VIRTIO_NET_F_MRG_RXBUF (1<<15) +#define VIRTIO_NET_F_STATUS (1<<16) +#define VIRTIO_NET_F_CTRL_VQ (1<<17) +#define VIRTIO_NET_F_CTRL_RX (1<<18) +#define VIRTIO_NET_F_CTRL_VLAN (1<<19) +#define VIRTIO_NET_F_CTRL_RX_EXTRA (1<<20) +#define VIRTIO_NET_F_GUEST_ANNOUNCE (1<<21) +#define VIRTIO_NET_F_MQ (1<<22) #define VIRTIO_NET_FLAG_BITS \ VIRTIO_COMMON_FLAG_BITS \ + "\x17""MQ" \ + "\x16""GUEST_ANNOUNCE" \ + "\x15""CTRL_RX_EXTRA" \ "\x14""CTRL_VLAN" \ "\x13""CTRL_RX" \ "\x12""CTRL_VQ" \ @@ -152,6 +160,11 @@ struct virtio_net_ctrl_cmd { # define VIRTIO_NET_CTRL_VLAN_ADD 0 # define VIRTIO_NET_CTRL_VLAN_DEL 1 +#define VIRTIO_NET_CTRL_MQ 4 +# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 +# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 +# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 + struct virtio_net_ctrl_status { uint8_t ack; } __packed; @@ -171,73 +184,110 @@ struct virtio_net_ctrl_vlan { uint16_t id; } __packed; +struct virtio_net_ctrl_mq { + uint16_t virtqueue_pairs; +} __packed; + +struct vioif_ctrl_cmdspec { + bus_dmamap_t dmamap; + void *buf; + bus_size_t bufsize; +}; /* * if_vioifvar.h: */ +struct vioif_txqueue { + struct virtqueue *txq_vq; + kmutex_t *txq_lock; + bool txq_stopping; + bool txq_link_active; + pcq_t *txq_intrq; + + struct virtio_net_hdr *txq_hdrs; + bus_dmamap_t *txq_hdr_dmamaps; + + struct mbuf **txq_mbufs; + bus_dmamap_t *txq_dmamaps; + + void *txq_deferred_transmit; +}; + +struct vioif_rxqueue { + struct virtqueue *rxq_vq; + kmutex_t *rxq_lock; + bool rxq_stopping; + + struct virtio_net_hdr *rxq_hdrs; + bus_dmamap_t *rxq_hdr_dmamaps; + + struct mbuf **rxq_mbufs; + bus_dmamap_t *rxq_dmamaps; + + void *rxq_softint; +}; + +struct vioif_ctrlqueue { + struct virtqueue *ctrlq_vq; + enum { + FREE, INUSE, DONE + } ctrlq_inuse; + kcondvar_t ctrlq_wait; + kmutex_t ctrlq_wait_lock; + + struct virtio_net_ctrl_cmd *ctrlq_cmd; + struct virtio_net_ctrl_status *ctrlq_status; + struct virtio_net_ctrl_rx *ctrlq_rx; + struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; + struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; + struct virtio_net_ctrl_mq *ctrlq_mq; + + bus_dmamap_t ctrlq_cmd_dmamap; + bus_dmamap_t ctrlq_status_dmamap; + bus_dmamap_t ctrlq_rx_dmamap; + bus_dmamap_t ctrlq_tbl_uc_dmamap; + bus_dmamap_t ctrlq_tbl_mc_dmamap; + bus_dmamap_t ctrlq_mq_dmamap; +}; + struct vioif_softc { device_t sc_dev; struct virtio_softc *sc_virtio; - struct virtqueue sc_vq[3]; -#define VQ_RX 0 -#define VQ_TX 1 -#define VQ_CTRL 2 + struct virtqueue *sc_vqs; + + int sc_max_nvq_pairs; + int sc_req_nvq_pairs; + int sc_act_nvq_pairs; uint8_t sc_mac[ETHER_ADDR_LEN]; struct ethercom sc_ethercom; short sc_deferred_init_done; bool sc_link_active; - /* bus_dmamem */ - bus_dma_segment_t sc_hdr_segs[1]; - struct virtio_net_hdr *sc_hdrs; -#define sc_rx_hdrs sc_hdrs - struct virtio_net_hdr *sc_tx_hdrs; - struct virtio_net_ctrl_cmd *sc_ctrl_cmd; - struct virtio_net_ctrl_status *sc_ctrl_status; - struct virtio_net_ctrl_rx *sc_ctrl_rx; - struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; - struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; - - /* kmem */ - bus_dmamap_t *sc_arrays; -#define sc_rxhdr_dmamaps sc_arrays - bus_dmamap_t *sc_txhdr_dmamaps; - bus_dmamap_t *sc_rx_dmamaps; - bus_dmamap_t *sc_tx_dmamaps; - struct mbuf **sc_rx_mbufs; - struct mbuf **sc_tx_mbufs; - - bus_dmamap_t sc_ctrl_cmd_dmamap; - bus_dmamap_t sc_ctrl_status_dmamap; - bus_dmamap_t sc_ctrl_rx_dmamap; - bus_dmamap_t sc_ctrl_tbl_uc_dmamap; - bus_dmamap_t sc_ctrl_tbl_mc_dmamap; - - void *sc_rx_softint; - void *sc_ctl_softint; - - enum { - FREE, INUSE, DONE - } sc_ctrl_inuse; - kcondvar_t sc_ctrl_wait; - kmutex_t sc_ctrl_wait_lock; - kmutex_t sc_tx_lock; - kmutex_t sc_rx_lock; - bool sc_stopping; + struct vioif_txqueue *sc_txq; + struct vioif_rxqueue *sc_rxq; bool sc_has_ctrl; + struct vioif_ctrlqueue sc_ctrlq; + + bus_dma_segment_t sc_hdr_segs[1]; + void *sc_dmamem; + void *sc_kmem; + + void *sc_ctl_softint; }; #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ -#define VIOIF_TX_LOCK(_sc) mutex_enter(&(_sc)->sc_tx_lock) -#define VIOIF_TX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_tx_lock) -#define VIOIF_TX_LOCKED(_sc) mutex_owned(&(_sc)->sc_tx_lock) -#define VIOIF_RX_LOCK(_sc) mutex_enter(&(_sc)->sc_rx_lock) -#define VIOIF_RX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_rx_lock) -#define VIOIF_RX_LOCKED(_sc) mutex_owned(&(_sc)->sc_rx_lock) +#define VIOIF_TXQ_LOCK(_q) mutex_enter((_q)->txq_lock) +#define VIOIF_TXQ_TRYLOCK(_q) mutex_tryenter((_q)->txq_lock) +#define VIOIF_TXQ_UNLOCK(_q) mutex_exit((_q)->txq_lock) +#define VIOIF_TXQ_LOCKED(_q) mutex_owned((_q)->txq_lock) + +#define VIOIF_RXQ_LOCK(_q) mutex_enter((_q)->rxq_lock) +#define VIOIF_RXQ_UNLOCK(_q) mutex_exit((_q)->rxq_lock) +#define VIOIF_RXQ_LOCKED(_q) mutex_owned((_q)->rxq_lock) /* cfattach interface functions */ static int vioif_match(device_t, cfdata_t, void *); @@ -248,24 +298,28 @@ static void vioif_deferred_init(device_t); static int vioif_init(struct ifnet *); static void vioif_stop(struct ifnet *, int); static void vioif_start(struct ifnet *); +static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); +static int vioif_transmit(struct ifnet *, struct mbuf *); +static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); static int vioif_ioctl(struct ifnet *, u_long, void *); static void vioif_watchdog(struct ifnet *); /* rx */ -static int vioif_add_rx_mbuf(struct vioif_softc *, int); -static void vioif_free_rx_mbuf(struct vioif_softc *, int); -static void vioif_populate_rx_mbufs(struct vioif_softc *); -static void vioif_populate_rx_mbufs_locked(struct vioif_softc *); -static int vioif_rx_deq(struct vioif_softc *); -static int vioif_rx_deq_locked(struct vioif_softc *); +static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); +static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); +static void vioif_populate_rx_mbufs(struct vioif_rxqueue *); +static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *); +static int vioif_rx_deq(struct vioif_rxqueue *); +static int vioif_rx_deq_locked(struct vioif_rxqueue *); static int vioif_rx_vq_done(struct virtqueue *); static void vioif_rx_softint(void *); -static void vioif_rx_drain(struct vioif_softc *); +static void vioif_rx_drain(struct vioif_rxqueue *); /* tx */ static int vioif_tx_vq_done(struct virtqueue *); static int vioif_tx_vq_done_locked(struct virtqueue *); -static void vioif_tx_drain(struct vioif_softc *); +static void vioif_tx_drain(struct vioif_txqueue *); +static void vioif_deferred_transmit(void *); /* other control */ static bool vioif_is_link_up(struct vioif_softc *); @@ -278,6 +332,9 @@ static int vioif_rx_filter(struct vioif_softc *); static int vioif_ctrl_vq_done(struct virtqueue *); static int vioif_config_change(struct virtio_softc *); static void vioif_ctl_softint(void *); +static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); +static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); +static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), vioif_match, vioif_attach, NULL, NULL); @@ -293,45 +350,113 @@ vioif_match(device_t parent, cfdata_t match, void *aux) return 0; } +static int +vioif_alloc_queues(struct vioif_softc *sc) +{ + int nvq_pairs = sc->sc_max_nvq_pairs; + int nvqs = nvq_pairs * 2; + int i; + + sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, + KM_NOSLEEP); + if (sc->sc_rxq == NULL) + return -1; + + sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, + KM_NOSLEEP); + if (sc->sc_txq == NULL) + return -1; + + if (sc->sc_has_ctrl) + nvqs++; + + sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_NOSLEEP); + if (sc->sc_vqs == NULL) + return -1; + + nvqs = 0; + for (i = 0; i < nvq_pairs; i++) { + sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; + sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; + } + + if (sc->sc_has_ctrl) + sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; + + return 0; +} + +static void +vioif_free_queues(struct vioif_softc *sc) +{ + int nvq_pairs = sc->sc_max_nvq_pairs; + int nvqs = nvq_pairs * 2; + + if (sc->sc_ctrlq.ctrlq_vq) + nvqs++; + + if (sc->sc_txq) { + kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); + sc->sc_txq = NULL; + } + + if (sc->sc_rxq) { + kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); + sc->sc_rxq = NULL; + } + + if (sc->sc_vqs) { + kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); + sc->sc_vqs = NULL; + } +} + /* allocate memory */ /* * dma memory is used for: - * sc_rx_hdrs[slot]: metadata array for received frames (READ) - * sc_tx_hdrs[slot]: metadata array for frames to be sent (WRITE) - * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) - * sc_ctrl_status: return value for a command via ctrl vq (READ) - * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command + * rxq_hdrs[slot]: metadata array for received frames (READ) + * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) + * ctrlq_cmd: command to be sent via ctrl vq (WRITE) + * ctrlq_status: return value for a command via ctrl vq (READ) + * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command * (WRITE) - * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC * class command (WRITE) - * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC * class command (WRITE) - * sc_ctrl_* structures are allocated only one each; they are protected by - * sc_ctrl_inuse variable and sc_ctrl_wait condvar. + * ctrlq_* structures are allocated only one each; they are protected by + * ctrlq_inuse variable and ctrlq_wait condvar. */ /* * dynamically allocated memory is used for: - * sc_rxhdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] - * sc_txhdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] - * sc_rx_dmamaps[slot]: bus_dmamap_t array for received payload - * sc_tx_dmamaps[slot]: bus_dmamap_t array for sent payload - * sc_rx_mbufs[slot]: mbuf pointer array for received frames - * sc_tx_mbufs[slot]: mbuf pointer array for sent frames + * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] + * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] + * rxq_dmamaps[slot]: bus_dmamap_t array for received payload + * txq_dmamaps[slot]: bus_dmamap_t array for sent payload + * rxq_mbufs[slot]: mbuf pointer array for received frames + * txq_mbufs[slot]: mbuf pointer array for sent frames */ static int vioif_alloc_mems(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; - int allocsize, allocsize2, r, rsegs, i; + struct vioif_txqueue *txq; + struct vioif_rxqueue *rxq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + int allocsize, allocsize2, r, rsegs, i, qid; void *vaddr; intptr_t p; - int rxqsize, txqsize; - rxqsize = sc->sc_vq[VQ_RX].vq_num; - txqsize = sc->sc_vq[VQ_TX].vq_num; + allocsize = 0; + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; - allocsize = sizeof(struct virtio_net_hdr) * rxqsize; - allocsize += sizeof(struct virtio_net_hdr) * txqsize; + allocsize += + sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num; + allocsize += + sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num; + } if (sc->sc_has_ctrl) { allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; allocsize += sizeof(struct virtio_net_ctrl_status) * 1; @@ -339,6 +464,7 @@ vioif_alloc_mems(struct vioif_softc *sc) allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) + sizeof(struct virtio_net_ctrl_mac_tbl) + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; + allocsize += sizeof(struct virtio_net_ctrl_mq) * 1; } r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); @@ -357,65 +483,88 @@ vioif_alloc_mems(struct vioif_softc *sc) "error code %d\n", r); goto err_dmamem_alloc; } - sc->sc_hdrs = vaddr; + +#define P(p, p0, p0size) do { p0 = (void *) p; \ + p += p0size; } while (0) memset(vaddr, 0, allocsize); + sc->sc_dmamem = vaddr; p = (intptr_t) vaddr; - p += sizeof(struct virtio_net_hdr) * rxqsize; -#define P(name,size) do { sc->sc_ ##name = (void*) p; \ - p += size; } while (0) - P(tx_hdrs, sizeof(struct virtio_net_hdr) * txqsize); + + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; + + P(p, rxq->rxq_hdrs, + sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num); + P(p, txq->txq_hdrs, + sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num); + } if (sc->sc_has_ctrl) { - P(ctrl_cmd, sizeof(struct virtio_net_ctrl_cmd)); - P(ctrl_status, sizeof(struct virtio_net_ctrl_status)); - P(ctrl_rx, sizeof(struct virtio_net_ctrl_rx)); - P(ctrl_mac_tbl_uc, sizeof(struct virtio_net_ctrl_mac_tbl)); - P(ctrl_mac_tbl_mc, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); + P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd)); + P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status)); + P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx)); + P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0); + P(p, ctrlq->ctrlq_mac_tbl_mc, + (sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); + P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq)); + } + + allocsize2 = 0; + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + int rxqsize, txqsize; + + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; + rxqsize = rxq->rxq_vq->vq_num; + txqsize = txq->txq_vq->vq_num; + + allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; + allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; + allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; + + allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; + allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; + allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; + } + vaddr = kmem_zalloc(allocsize2, KM_SLEEP); + sc->sc_kmem = vaddr; + p = (intptr_t) vaddr; + + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + int rxqsize, txqsize; + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; + rxqsize = rxq->rxq_vq->vq_num; + txqsize = txq->txq_vq->vq_num; + + P(p, rxq->rxq_hdr_dmamaps, sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); + P(p, txq->txq_hdr_dmamaps, sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); + P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize); + P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize); + P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize); + P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize); } #undef P - allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize); - allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize); - allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize); - sc->sc_arrays = kmem_zalloc(allocsize2, KM_SLEEP); - sc->sc_txhdr_dmamaps = sc->sc_arrays + rxqsize; - sc->sc_rx_dmamaps = sc->sc_txhdr_dmamaps + txqsize; - sc->sc_tx_dmamaps = sc->sc_rx_dmamaps + rxqsize; - sc->sc_rx_mbufs = (void*) (sc->sc_tx_dmamaps + txqsize); - sc->sc_tx_mbufs = sc->sc_rx_mbufs + rxqsize; - -#define C(map, buf, size, nsegs, rw, usage) \ - do { \ - r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ - BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ - &sc->sc_ ##map); \ - if (r != 0) { \ - aprint_error_dev(sc->sc_dev, \ - usage " dmamap creation failed, " \ - "error code %d\n", r); \ - goto err_reqs; \ - } \ - } while (0) -#define C_L1(map, buf, size, nsegs, rw, usage) \ - C(map, buf, size, nsegs, rw, usage); \ - do { \ - r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ - &sc->sc_ ##buf, size, NULL, \ - BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ - if (r != 0) { \ - aprint_error_dev(sc->sc_dev, \ - usage " dmamap load failed, " \ - "error code %d\n", r); \ - goto err_reqs; \ - } \ +#define C(map, size, nsegs, usage) \ + do { \ + r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ + BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ + &map); \ + if (r != 0) { \ + aprint_error_dev(sc->sc_dev, \ + "%s dmamap creation failed, " \ + "error code %d\n", usage, r); \ + goto err_reqs; \ + } \ } while (0) -#define C_L2(map, buf, size, nsegs, rw, usage) \ - C(map, buf, size, nsegs, rw, usage); \ +#define C_L(map, buf, size, nsegs, rw, usage) \ + C(map, size, nsegs, usage); \ do { \ - r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ - sc->sc_ ##buf, size, NULL, \ - BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ + r = bus_dmamap_load(virtio_dmat(vsc), map, \ + buf, size, NULL, \ + rw | BUS_DMA_NOWAIT); \ if (r != 0) { \ aprint_error_dev(sc->sc_dev, \ usage " dmamap load failed, " \ @@ -423,51 +572,59 @@ vioif_alloc_mems(struct vioif_softc *sc) goto err_reqs; \ } \ } while (0) - for (i = 0; i < rxqsize; i++) { - C_L1(rxhdr_dmamaps[i], rx_hdrs[i], - sizeof(struct virtio_net_hdr), 1, - READ, "rx header"); - C(rx_dmamaps[i], NULL, MCLBYTES, 1, 0, "rx payload"); - } - for (i = 0; i < txqsize; i++) { - C_L1(txhdr_dmamaps[i], tx_hdrs[i], - sizeof(struct virtio_net_hdr), 1, - WRITE, "tx header"); - C(tx_dmamaps[i], NULL, ETHER_MAX_LEN, VIRTIO_NET_TX_MAXNSEGS, 0, - "tx payload"); + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; + + for (i = 0; i < rxq->rxq_vq->vq_num; i++) { + C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i], + sizeof(rxq->rxq_hdrs[0]), 1, + BUS_DMA_READ, "rx header"); + C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload"); + } + + for (i = 0; i < txq->txq_vq->vq_num; i++) { + C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i], + sizeof(txq->txq_hdrs[0]), 1, + BUS_DMA_READ, "tx header"); + C(txq->txq_dmamaps[i], ETHER_MAX_LEN, + VIRTIO_NET_TX_MAXNSEGS, "tx payload"); + } } if (sc->sc_has_ctrl) { /* control vq class & command */ - C_L2(ctrl_cmd_dmamap, ctrl_cmd, - sizeof(struct virtio_net_ctrl_cmd), 1, WRITE, - "control command"); - - /* control vq status */ - C_L2(ctrl_status_dmamap, ctrl_status, - sizeof(struct virtio_net_ctrl_status), 1, READ, - "control status"); + C_L(ctrlq->ctrlq_cmd_dmamap, + ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, + BUS_DMA_WRITE, "control command"); + C_L(ctrlq->ctrlq_status_dmamap, + ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, + BUS_DMA_READ, "control status"); /* control vq rx mode command parameter */ - C_L2(ctrl_rx_dmamap, ctrl_rx, - sizeof(struct virtio_net_ctrl_rx), 1, WRITE, - "rx mode control command"); + C_L(ctrlq->ctrlq_rx_dmamap, + ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, + BUS_DMA_WRITE, "rx mode control command"); + + /* multiqueue set command */ + C_L(ctrlq->ctrlq_mq_dmamap, + ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, + BUS_DMA_WRITE, "multiqueue set command"); /* control vq MAC filter table for unicast */ /* do not load now since its length is variable */ - C(ctrl_tbl_uc_dmamap, NULL, - sizeof(struct virtio_net_ctrl_mac_tbl) + 0, 1, WRITE, - "unicast MAC address filter command"); + C(ctrlq->ctrlq_tbl_uc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1, + "unicast MAC address filter command"); /* control vq MAC filter table for multicast */ - C(ctrl_tbl_mc_dmamap, NULL, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES), - 1, WRITE, "multicast MAC address filter command"); + C(ctrlq->ctrlq_tbl_mc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, + "multicast MAC address filter command"); } -#undef C_L2 -#undef C_L1 +#undef C_L #undef C return 0; @@ -475,30 +632,35 @@ vioif_alloc_mems(struct vioif_softc *sc) err_reqs: #define D(map) \ do { \ - if (sc->sc_ ##map) { \ - bus_dmamap_destroy(virtio_dmat(vsc), sc->sc_ ##map); \ - sc->sc_ ##map = NULL; \ + if (map) { \ + bus_dmamap_destroy(virtio_dmat(vsc), map); \ + map = NULL; \ } \ } while (0) - D(ctrl_tbl_mc_dmamap); - D(ctrl_tbl_uc_dmamap); - D(ctrl_rx_dmamap); - D(ctrl_status_dmamap); - D(ctrl_cmd_dmamap); - for (i = 0; i < txqsize; i++) { - D(tx_dmamaps[i]); - D(txhdr_dmamaps[i]); - } - for (i = 0; i < rxqsize; i++) { - D(rx_dmamaps[i]); - D(rxhdr_dmamaps[i]); + D(ctrlq->ctrlq_tbl_mc_dmamap); + D(ctrlq->ctrlq_tbl_uc_dmamap); + D(ctrlq->ctrlq_rx_dmamap); + D(ctrlq->ctrlq_status_dmamap); + D(ctrlq->ctrlq_cmd_dmamap); + for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { + rxq = &sc->sc_rxq[qid]; + txq = &sc->sc_txq[qid]; + + for (i = 0; i < txq->txq_vq->vq_num; i++) { + D(txq->txq_dmamaps[i]); + D(txq->txq_hdr_dmamaps[i]); + } + for (i = 0; i < rxq->rxq_vq->vq_num; i++) { + D(rxq->rxq_dmamaps[i]); + D(rxq->rxq_hdr_dmamaps[i]); + } } #undef D - if (sc->sc_arrays) { - kmem_free(sc->sc_arrays, allocsize2); - sc->sc_arrays = 0; + if (sc->sc_kmem) { + kmem_free(sc->sc_kmem, allocsize2); + sc->sc_kmem = NULL; } - bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_hdrs, allocsize); + bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); err_dmamem_alloc: bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); err_none: @@ -510,10 +672,13 @@ vioif_attach(device_t parent, device_t self, void *aux) { struct vioif_softc *sc = device_private(self); struct virtio_softc *vsc = device_private(parent); - uint32_t features; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + struct vioif_txqueue *txq; + struct vioif_rxqueue *rxq; + uint32_t features, req_features; struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int flags; - int r, nvqs=0, req_flags; + u_int softint_flags; + int r, i, nvqs=0, req_flags; if (virtio_child(vsc) != NULL) { aprint_normal(": child already attached for %s; " @@ -526,6 +691,10 @@ vioif_attach(device_t parent, device_t self, void *aux) sc->sc_virtio = vsc; sc->sc_link_active = false; + sc->sc_max_nvq_pairs = 1; + sc->sc_req_nvq_pairs = 1; + sc->sc_act_nvq_pairs = 1; + req_flags = 0; #ifdef VIOIF_MPSAFE @@ -536,11 +705,15 @@ vioif_attach(device_t parent, device_t self, void *aux) #endif req_flags |= VIRTIO_F_PCI_INTR_MSIX; - virtio_child_attach_start(vsc, self, IPL_NET, sc->sc_vq, + req_features = + VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | + VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; +#ifdef VIOIF_MULTIQ + req_features |= VIRTIO_NET_F_MQ; +#endif + virtio_child_attach_start(vsc, self, IPL_NET, NULL, vioif_config_change, virtio_vq_intr, req_flags, - (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | - VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY), - VIRTIO_NET_FLAG_BITS); + req_features, VIRTIO_NET_FLAG_BITS); features = virtio_features(vsc); @@ -586,69 +759,109 @@ vioif_attach(device_t parent, device_t self, void *aux) aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); - mutex_init(&sc->sc_tx_lock, MUTEX_DEFAULT, IPL_NET); - mutex_init(&sc->sc_rx_lock, MUTEX_DEFAULT, IPL_NET); - sc->sc_stopping = false; + if ((features & VIRTIO_NET_F_CTRL_VQ) && + (features & VIRTIO_NET_F_CTRL_RX)) { + sc->sc_has_ctrl = true; - /* - * Allocating a virtqueue for Rx - */ - r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_RX], VQ_RX, - MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); + cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); + mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); + ctrlq->ctrlq_inuse = FREE; + } else { + sc->sc_has_ctrl = false; + } + + if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { + sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, + VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); + + /* Limit the number of queue pairs to use */ + if (sc->sc_max_nvq_pairs <= ncpu) + sc->sc_req_nvq_pairs = sc->sc_max_nvq_pairs; + else + sc->sc_req_nvq_pairs = ncpu; + } + + r = vioif_alloc_queues(sc); if (r != 0) goto err; - nvqs = 1; - sc->sc_vq[VQ_RX].vq_done = vioif_rx_vq_done; + + virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); + +#ifdef VIOIF_MPSAFE + softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; +#else + softint_flags = SOFTINT_NET; +#endif /* - * Allocating a virtqueue for Tx + * Allocating a virtqueues */ - r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_TX], VQ_TX, - (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), - VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); - if (r != 0) - goto err; - nvqs = 2; - sc->sc_vq[VQ_TX].vq_done = vioif_tx_vq_done; + for (i = 0; i < sc->sc_max_nvq_pairs; i++) { + rxq = &sc->sc_rxq[i]; + txq = &sc->sc_txq[i]; + char qname[32]; - virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); - virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); /* not urgent; do it later */ + rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); - if ((features & VIRTIO_NET_F_CTRL_VQ) && - (features & VIRTIO_NET_F_CTRL_RX)) { + rxq->rxq_softint = softint_establish(softint_flags, vioif_rx_softint, rxq); + if (rxq->rxq_softint == NULL) { + aprint_error_dev(self, "cannot establish rx softint\n"); + goto err; + } + snprintf(qname, sizeof(qname), "rx%d", i); + r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, + MCLBYTES+sizeof(struct virtio_net_hdr), nvqs, qname); + if (r != 0) + goto err; + nvqs++; + rxq->rxq_vq->vq_done = vioif_rx_vq_done; + rxq->rxq_vq->vq_done_ctx = (void *)rxq; + rxq->rxq_stopping = true; + + txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); + txq->txq_deferred_transmit = softint_establish(softint_flags, + vioif_deferred_transmit, txq); + if (txq->txq_deferred_transmit == NULL) { + aprint_error_dev(self, "cannot establish tx softint\n"); + goto err; + } + snprintf(qname, sizeof(qname), "tx%d", i); + r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, + (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), + VIRTIO_NET_TX_MAXNSEGS + 1, qname); + if (r != 0) + goto err; + nvqs++; + txq->txq_vq->vq_done = vioif_tx_vq_done; + txq->txq_vq->vq_done_ctx = (void *)txq; + txq->txq_link_active = sc->sc_link_active; + txq->txq_stopping = false; + txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_NOSLEEP); + if (txq->txq_intrq == NULL) + goto err; + } + + if (sc->sc_has_ctrl) { /* * Allocating a virtqueue for control channel */ - r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_CTRL], VQ_CTRL, + r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, NBPG, 1, "control"); if (r != 0) { aprint_error_dev(self, "failed to allocate " "a virtqueue for control channel\n"); - goto skip; - } - - sc->sc_vq[VQ_CTRL].vq_done = vioif_ctrl_vq_done; - cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); - mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); - sc->sc_ctrl_inuse = FREE; - virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); - sc->sc_has_ctrl = true; - nvqs = 3; - } -skip: -#ifdef VIOIF_MPSAFE - flags = SOFTINT_NET | SOFTINT_MPSAFE; -#else - flags = SOFTINT_NET; -#endif - sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc); - if (sc->sc_rx_softint == NULL) { - aprint_error_dev(self, "cannot establish rx softint\n"); - goto err; + sc->sc_has_ctrl = false; + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); + } else { + nvqs++; + ctrlq->ctrlq_vq->vq_done = vioif_ctrl_vq_done; + ctrlq->ctrlq_vq->vq_done_ctx = (void *) ctrlq; + } } - sc->sc_ctl_softint = softint_establish(flags, vioif_ctl_softint, sc); + sc->sc_ctl_softint = softint_establish(softint_flags, vioif_ctl_softint, sc); if (sc->sc_ctl_softint == NULL) { aprint_error_dev(self, "cannot establish ctl softint\n"); goto err; @@ -663,13 +876,19 @@ skip: strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; +#ifdef VIOIF_MPSAFE + ifp->if_extflags = IFEF_MPSAFE; +#endif ifp->if_start = vioif_start; + if (sc->sc_req_nvq_pairs > 1) + ifp->if_transmit = vioif_transmit; ifp->if_ioctl = vioif_ioctl; ifp->if_init = vioif_init; ifp->if_stop = vioif_stop; ifp->if_capabilities = 0; ifp->if_watchdog = vioif_watchdog; - IFQ_SET_MAXLEN(&ifp->if_snd, MAX(sc->sc_vq[VQ_TX].vq_num, IFQ_MAXLEN)); + txq = &sc->sc_txq[0]; + IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); IFQ_SET_READY(&ifp->if_snd); sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; @@ -681,16 +900,45 @@ skip: return; err: - mutex_destroy(&sc->sc_tx_lock); - mutex_destroy(&sc->sc_rx_lock); + for (i = 0; i < sc->sc_max_nvq_pairs; i++) { + rxq = &sc->sc_rxq[i]; + txq = &sc->sc_txq[i]; + + if (rxq->rxq_lock) { + mutex_obj_free(rxq->rxq_lock); + rxq->rxq_lock = NULL; + } + + if (rxq->rxq_softint) { + softint_disestablish(rxq->rxq_softint); + rxq->rxq_softint = NULL; + } + + if (txq->txq_lock) { + mutex_obj_free(txq->txq_lock); + txq->txq_lock = NULL; + } + + if (txq->txq_deferred_transmit) { + softint_disestablish(txq->txq_deferred_transmit); + txq->txq_deferred_transmit = NULL; + } + + if (txq->txq_intrq) { + pcq_destroy(txq->txq_intrq); + txq->txq_intrq = NULL; + } + } if (sc->sc_has_ctrl) { - cv_destroy(&sc->sc_ctrl_wait); - mutex_destroy(&sc->sc_ctrl_wait_lock); + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); } while (nvqs > 0) - virtio_free_vq(vsc, &sc->sc_vq[--nvqs]); + virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); + + vioif_free_queues(sc); virtio_child_attach_failed(vsc); return; @@ -713,6 +961,40 @@ vioif_deferred_init(device_t self) "errror code %d\n", r); } +static void +vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) +{ + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_txqueue *txq; + struct vioif_rxqueue *rxq; + int i; + + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + txq = &sc->sc_txq[i]; + rxq = &sc->sc_rxq[i]; + + virtio_start_vq_intr(vsc, txq->txq_vq); + virtio_start_vq_intr(vsc, rxq->rxq_vq); + } +} + +static void +vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) +{ + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_txqueue *txq; + struct vioif_rxqueue *rxq; + int i; + + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + txq = &sc->sc_txq[i]; + rxq = &sc->sc_rxq[i]; + + virtio_stop_vq_intr(vsc, txq->txq_vq); + virtio_stop_vq_intr(vsc, rxq->rxq_vq); + } +} + /* * Interface functions for ifnet */ @@ -721,16 +1003,36 @@ vioif_init(struct ifnet *ifp) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_rxqueue *rxq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + int r, i; vioif_stop(ifp, 0); virtio_reinit_start(vsc); virtio_negotiate_features(vsc, virtio_features(vsc)); - virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); - virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); - if (sc->sc_has_ctrl) - virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); + + for (i = 0; i < sc->sc_req_nvq_pairs; i++) { + rxq = &sc->sc_rxq[i]; + + /* Have to set false before vioif_populate_rx_mbufs */ + rxq->rxq_stopping = false; + vioif_populate_rx_mbufs(rxq); + } + virtio_reinit_end(vsc); + virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); + + r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); + if (r == 0) + sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; + else + sc->sc_act_nvq_pairs = 1; + + for (i = 0; i < sc->sc_act_nvq_pairs; i++) + sc->sc_txq[i].txq_stopping = false; + + vioif_enable_interrupt_vqpairs(sc); if (!sc->sc_deferred_init_done) { sc->sc_deferred_init_done = 1; @@ -738,11 +1040,6 @@ vioif_init(struct ifnet *ifp) vioif_deferred_init(sc->sc_dev); } - /* Have to set false before vioif_populate_rx_mbufs */ - sc->sc_stopping = false; - - vioif_populate_rx_mbufs(sc); - vioif_update_link_status(sc); ifp->if_flags |= IFF_RUNNING; ifp->if_flags &= ~IFF_OACTIVE; @@ -756,53 +1053,80 @@ vioif_stop(struct ifnet *ifp, int disable) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_txqueue *txq; + struct vioif_rxqueue *rxq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + int i; /* Take the locks to ensure that ongoing TX/RX finish */ - VIOIF_TX_LOCK(sc); - VIOIF_RX_LOCK(sc); - sc->sc_stopping = true; - VIOIF_RX_UNLOCK(sc); - VIOIF_TX_UNLOCK(sc); + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + txq = &sc->sc_txq[i]; + rxq = &sc->sc_rxq[i]; + + VIOIF_TXQ_LOCK(txq); + txq->txq_stopping = true; + VIOIF_TXQ_UNLOCK(txq); + + VIOIF_RXQ_LOCK(rxq); + rxq->rxq_stopping = true; + VIOIF_RXQ_UNLOCK(rxq); + } /* disable interrupts */ - virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_RX]); - virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); + vioif_disable_interrupt_vqpairs(sc); + if (sc->sc_has_ctrl) - virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); + virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); /* only way to stop I/O and DMA is resetting... */ virtio_reset(vsc); - vioif_rx_deq(sc); - vioif_tx_drain(sc); + for (i = 0; i < sc->sc_act_nvq_pairs; i++) + vioif_rx_deq(&sc->sc_rxq[i]); + ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); sc->sc_link_active = false; - if (disable) - vioif_rx_drain(sc); + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + txq = &sc->sc_txq[i]; + rxq = &sc->sc_rxq[i]; + + txq->txq_link_active = false; + + if (disable) + vioif_rx_drain(rxq); + + vioif_tx_drain(txq); + } } static void -vioif_start(struct ifnet *ifp) +vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, bool is_transmit) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = &sc->sc_vq[VQ_TX]; + struct virtqueue *vq = txq->txq_vq; struct mbuf *m; int queued = 0; - VIOIF_TX_LOCK(sc); + KASSERT(VIOIF_TXQ_LOCKED(txq)); - if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING || - !sc->sc_link_active) - goto out; + if ((ifp->if_flags & IFF_RUNNING) == 0) + return; - if (sc->sc_stopping) - goto out; + if (!txq->txq_link_active || txq->txq_stopping) + return; + + if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) + return; for (;;) { int slot, r; - IFQ_DEQUEUE(&ifp->if_snd, m); + if (is_transmit) + m = pcq_get(txq->txq_intrq); + else + IFQ_DEQUEUE(&ifp->if_snd, m); + if (m == NULL) break; @@ -816,7 +1140,7 @@ vioif_start(struct ifnet *ifp) panic("enqueue_prep for a tx buffer"); r = bus_dmamap_load_mbuf(virtio_dmat(vsc), - sc->sc_tx_dmamaps[slot], + txq->txq_dmamaps[slot], m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); if (r != 0) { /* maybe just too fragmented */ @@ -831,7 +1155,7 @@ vioif_start(struct ifnet *ifp) m = newm; r = bus_dmamap_load_mbuf(virtio_dmat(vsc), - sc->sc_tx_dmamaps[slot], + txq->txq_dmamaps[slot], m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); if (r != 0) { aprint_error_dev(sc->sc_dev, @@ -846,29 +1170,29 @@ skip: /* This should actually never fail */ r = virtio_enqueue_reserve(vsc, vq, slot, - sc->sc_tx_dmamaps[slot]->dm_nsegs + 1); + txq->txq_dmamaps[slot]->dm_nsegs + 1); if (r != 0) { aprint_error_dev(sc->sc_dev, "virtio_enqueue_reserve failed, error code %d\n", r); bus_dmamap_unload(virtio_dmat(vsc), - sc->sc_tx_dmamaps[slot]); + txq->txq_dmamaps[slot]); /* slot already freed by virtio_enqueue_reserve */ m_freem(m); continue; } - sc->sc_tx_mbufs[slot] = m; + txq->txq_mbufs[slot] = m; - memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], - 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, + memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); + bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], + 0, txq->txq_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], - 0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize, + bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], + 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_PREWRITE); - virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true); - virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true); + virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); + virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); virtio_enqueue_commit(vsc, vq, slot, false); queued++; @@ -879,9 +1203,91 @@ skip: virtio_enqueue_commit(vsc, vq, -1, true); ifp->if_timer = 5; } +} -out: - VIOIF_TX_UNLOCK(sc); +static void +vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) +{ + + /* + * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. + */ + vioif_send_common_locked(ifp, txq, false); + +} + +static void +vioif_start(struct ifnet *ifp) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_txqueue *txq = &sc->sc_txq[0]; + +#ifdef VIOIF_MPSAFE + KASSERT(if_is_mpsafe(ifp)); +#endif + + VIOIF_TXQ_LOCK(txq); + if (!txq->txq_stopping) + vioif_start_locked(ifp, txq); + VIOIF_TXQ_UNLOCK(txq); +} + +static inline int +vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + u_int cpuid = cpu_index(curcpu()); + + return cpuid % sc->sc_act_nvq_pairs; +} + +static void +vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) +{ + + vioif_send_common_locked(ifp, txq, true); +} + +static int +vioif_transmit(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_txqueue *txq; + int qid; + + qid = vioif_select_txqueue(ifp, m); + txq = &sc->sc_txq[qid]; + + if (__predict_false(!pcq_put(txq->txq_intrq, m))) { + m_freem(m); + return ENOBUFS; + } + + ifp->if_obytes += m->m_pkthdr.len; + if (m->m_flags & M_MCAST) + ifp->if_omcasts++; + + if (VIOIF_TXQ_TRYLOCK(txq)) { + if (!txq->txq_stopping) + vioif_transmit_locked(ifp, txq); + VIOIF_TXQ_UNLOCK(txq); + } + + return 0; +} + +static void +vioif_deferred_transmit(void *arg) +{ + struct vioif_txqueue *txq = arg; + struct virtio_softc *vsc = txq->txq_vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + + if (VIOIF_TXQ_TRYLOCK(txq)) { + vioif_send_common_locked(ifp, txq, true); + VIOIF_TXQ_UNLOCK(txq); + } } static int @@ -909,9 +1315,12 @@ void vioif_watchdog(struct ifnet *ifp) { struct vioif_softc *sc = ifp->if_softc; + int i; - if (ifp->if_flags & IFF_RUNNING) - vioif_tx_vq_done(&sc->sc_vq[VQ_TX]); + if (ifp->if_flags & IFF_RUNNING) { + for (i = 0; i < sc->sc_act_nvq_pairs; i++) + vioif_tx_vq_done(sc->sc_txq[i].txq_vq); + } } @@ -920,8 +1329,9 @@ vioif_watchdog(struct ifnet *ifp) */ /* allocate and initialize a mbuf for receive */ static int -vioif_add_rx_mbuf(struct vioif_softc *sc, int i) +vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) { + struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; struct mbuf *m; int r; @@ -933,14 +1343,14 @@ vioif_add_rx_mbuf(struct vioif_softc *sc, int i) m_freem(m); return ENOBUFS; } - sc->sc_rx_mbufs[i] = m; + rxq->rxq_mbufs[i] = m; m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; - r = bus_dmamap_load_mbuf(virtio_dmat(sc->sc_virtio), - sc->sc_rx_dmamaps[i], + r = bus_dmamap_load_mbuf(virtio_dmat(vsc), + rxq->rxq_dmamaps[i], m, BUS_DMA_READ|BUS_DMA_NOWAIT); if (r) { m_freem(m); - sc->sc_rx_mbufs[i] = 0; + rxq->rxq_mbufs[i] = NULL; return r; } @@ -949,32 +1359,36 @@ vioif_add_rx_mbuf(struct vioif_softc *sc, int i) /* free a mbuf for receive */ static void -vioif_free_rx_mbuf(struct vioif_softc *sc, int i) +vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) { - bus_dmamap_unload(virtio_dmat(sc->sc_virtio), sc->sc_rx_dmamaps[i]); - m_freem(sc->sc_rx_mbufs[i]); - sc->sc_rx_mbufs[i] = NULL; + struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; + + bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); + m_freem(rxq->rxq_mbufs[i]); + rxq->rxq_mbufs[i] = NULL; } /* add mbufs for all the empty receive slots */ static void -vioif_populate_rx_mbufs(struct vioif_softc *sc) +vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq) { - VIOIF_RX_LOCK(sc); - vioif_populate_rx_mbufs_locked(sc); - VIOIF_RX_UNLOCK(sc); + + VIOIF_RXQ_LOCK(rxq); + vioif_populate_rx_mbufs_locked(rxq); + VIOIF_RXQ_UNLOCK(rxq); } static void -vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) +vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq) { - struct virtio_softc *vsc = sc->sc_virtio; + struct virtqueue *vq = rxq->rxq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); int i, r, ndone = 0; - struct virtqueue *vq = &sc->sc_vq[VQ_RX]; - KASSERT(VIOIF_RX_LOCKED(sc)); + KASSERT(VIOIF_RXQ_LOCKED(rxq)); - if (sc->sc_stopping) + if (rxq->rxq_stopping) return; for (i = 0; i < vq->vq_num; i++) { @@ -984,8 +1398,8 @@ vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) break; if (r != 0) panic("enqueue_prep for rx buffers"); - if (sc->sc_rx_mbufs[slot] == NULL) { - r = vioif_add_rx_mbuf(sc, slot); + if (rxq->rxq_mbufs[slot] == NULL) { + r = vioif_add_rx_mbuf(rxq, slot); if (r != 0) { printf("%s: rx mbuf allocation failed, " "error code %d\n", @@ -994,17 +1408,17 @@ vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) } } r = virtio_enqueue_reserve(vsc, vq, slot, - sc->sc_rx_dmamaps[slot]->dm_nsegs + 1); + rxq->rxq_dmamaps[slot]->dm_nsegs + 1); if (r != 0) { - vioif_free_rx_mbuf(sc, slot); + vioif_free_rx_mbuf(rxq, slot); break; } - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], + bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], + bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 0, MCLBYTES, BUS_DMASYNC_PREREAD); - virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false); - virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false); + virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], false); + virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); virtio_enqueue_commit(vsc, vq, slot, false); ndone++; } @@ -1014,54 +1428,55 @@ vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) /* dequeue received packets */ static int -vioif_rx_deq(struct vioif_softc *sc) +vioif_rx_deq(struct vioif_rxqueue *rxq) { int r; - KASSERT(sc->sc_stopping); + KASSERT(rxq->rxq_stopping); - VIOIF_RX_LOCK(sc); - r = vioif_rx_deq_locked(sc); - VIOIF_RX_UNLOCK(sc); + VIOIF_RXQ_LOCK(rxq); + r = vioif_rx_deq_locked(rxq); + VIOIF_RXQ_UNLOCK(rxq); return r; } /* dequeue received packets */ static int -vioif_rx_deq_locked(struct vioif_softc *sc) +vioif_rx_deq_locked(struct vioif_rxqueue *rxq) { - struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = &sc->sc_vq[VQ_RX]; + struct virtqueue *vq = rxq->rxq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct mbuf *m; int r = 0; int slot, len; - KASSERT(VIOIF_RX_LOCKED(sc)); + KASSERT(VIOIF_RXQ_LOCKED(rxq)); while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { len -= sizeof(struct virtio_net_hdr); r = 1; - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], + bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], + bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 0, MCLBYTES, BUS_DMASYNC_POSTREAD); - m = sc->sc_rx_mbufs[slot]; + m = rxq->rxq_mbufs[slot]; KASSERT(m != NULL); - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot]); - sc->sc_rx_mbufs[slot] = 0; + bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); + rxq->rxq_mbufs[slot] = NULL; virtio_dequeue_commit(vsc, vq, slot); m_set_rcvif(m, ifp); m->m_len = m->m_pkthdr.len = len; - VIOIF_RX_UNLOCK(sc); + VIOIF_RXQ_UNLOCK(rxq); if_percpuq_enqueue(ifp->if_percpuq, m); - VIOIF_RX_LOCK(sc); + VIOIF_RXQ_LOCK(rxq); - if (sc->sc_stopping) + if (rxq->rxq_stopping) break; } @@ -1072,29 +1487,28 @@ vioif_rx_deq_locked(struct vioif_softc *sc) static int vioif_rx_vq_done(struct virtqueue *vq) { - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct vioif_rxqueue *rxq = vq->vq_done_ctx; int r = 0; #ifdef VIOIF_SOFTINT_INTR KASSERT(!cpu_intr_p()); #endif - VIOIF_RX_LOCK(sc); + VIOIF_RXQ_LOCK(rxq); - if (sc->sc_stopping) + if (rxq->rxq_stopping) goto out; - r = vioif_rx_deq_locked(sc); + r = vioif_rx_deq_locked(rxq); if (r) #ifdef VIOIF_SOFTINT_INTR - vioif_populate_rx_mbufs_locked(sc); + vioif_populate_rx_mbufs_locked(rxq); #else - softint_schedule(sc->sc_rx_softint); + softint_schedule(rxq->rxq_softint); #endif out: - VIOIF_RX_UNLOCK(sc); + VIOIF_RXQ_UNLOCK(rxq); return r; } @@ -1102,22 +1516,22 @@ out: static void vioif_rx_softint(void *arg) { - struct vioif_softc *sc = arg; + struct vioif_rxqueue *rxq = arg; - vioif_populate_rx_mbufs(sc); + vioif_populate_rx_mbufs(rxq); } /* free all the mbufs; called from if_stop(disable) */ static void -vioif_rx_drain(struct vioif_softc *sc) +vioif_rx_drain(struct vioif_rxqueue *rxq) { - struct virtqueue *vq = &sc->sc_vq[VQ_RX]; + struct virtqueue *vq = rxq->rxq_vq; int i; for (i = 0; i < vq->vq_num; i++) { - if (sc->sc_rx_mbufs[i] == NULL) + if (rxq->rxq_mbufs[i] == NULL) continue; - vioif_free_rx_mbuf(sc, i); + vioif_free_rx_mbuf(rxq, i); } } @@ -1137,19 +1551,24 @@ vioif_tx_vq_done(struct virtqueue *vq) struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_txqueue *txq = vq->vq_done_ctx; int r = 0; - VIOIF_TX_LOCK(sc); + VIOIF_TXQ_LOCK(txq); - if (sc->sc_stopping) + if (txq->txq_stopping) goto out; r = vioif_tx_vq_done_locked(vq); out: - VIOIF_TX_UNLOCK(sc); - if (r) + VIOIF_TXQ_UNLOCK(txq); + if (r) { if_schedule_deferred_start(ifp); + + KASSERT(txq->txq_deferred_transmit != NULL); + softint_schedule(txq->txq_deferred_transmit); + } return r; } @@ -1158,24 +1577,25 @@ vioif_tx_vq_done_locked(struct virtqueue *vq) { struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct vioif_txqueue *txq = vq->vq_done_ctx; struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct mbuf *m; int r = 0; int slot, len; - KASSERT(VIOIF_TX_LOCKED(sc)); + KASSERT(VIOIF_TXQ_LOCKED(txq)); while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { r++; - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], + bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], - 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, + bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], + 0, txq->txq_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_POSTWRITE); - m = sc->sc_tx_mbufs[slot]; - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot]); - sc->sc_tx_mbufs[slot] = 0; + m = txq->txq_mbufs[slot]; + bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); + txq->txq_mbufs[slot] = NULL; virtio_dequeue_commit(vsc, vq, slot); ifp->if_opackets++; m_freem(m); @@ -1188,20 +1608,20 @@ vioif_tx_vq_done_locked(struct virtqueue *vq) /* free all the mbufs already put on vq; called from if_stop(disable) */ static void -vioif_tx_drain(struct vioif_softc *sc) +vioif_tx_drain(struct vioif_txqueue *txq) { - struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = &sc->sc_vq[VQ_TX]; + struct virtqueue *vq = txq->txq_vq; + struct virtio_softc *vsc = vq->vq_owner; int i; - KASSERT(sc->sc_stopping); + KASSERT(txq->txq_stopping); for (i = 0; i < vq->vq_num; i++) { - if (sc->sc_tx_mbufs[i] == NULL) + if (txq->txq_mbufs[i] == NULL) continue; - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[i]); - m_freem(sc->sc_tx_mbufs[i]); - sc->sc_tx_mbufs[i] = NULL; + bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); + m_freem(txq->txq_mbufs[i]); + txq->txq_mbufs[i] = NULL; } } @@ -1209,65 +1629,129 @@ vioif_tx_drain(struct vioif_softc *sc) * Control vq */ /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ +static void +vioif_ctrl_acquire(struct vioif_softc *sc) +{ + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + + mutex_enter(&ctrlq->ctrlq_wait_lock); + while (ctrlq->ctrlq_inuse != FREE) + cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); + ctrlq->ctrlq_inuse = INUSE; + mutex_exit(&ctrlq->ctrlq_wait_lock); +} + +static void +vioif_ctrl_release(struct vioif_softc *sc) +{ + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + + mutex_enter(&ctrlq->ctrlq_wait_lock); + ctrlq->ctrlq_inuse = FREE; + cv_signal(&ctrlq->ctrlq_wait); + mutex_exit(&ctrlq->ctrlq_wait_lock); +} + static int -vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) +vioif_ctrl_load_cmdspec(struct vioif_softc *sc, + struct vioif_ctrl_cmdspec *specs, int nspecs) { struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; - int r, slot; + int i, r, loaded; + + loaded = 0; + for (i = 0; i < nspecs; i++) { + r = bus_dmamap_load(virtio_dmat(vsc), + specs[i].dmamap, specs[i].buf, specs[i].bufsize, + NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); + if (r) { + printf("%s: control command dmamap load failed, " + "error code %d\n", device_xname(sc->sc_dev), r); + goto err; + } + loaded++; - if (!sc->sc_has_ctrl) - return ENOTSUP; + } + + return r; + +err: + for (i = 0; i < loaded; i++) { + bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); + } + + return r; +} + +static void +vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, + struct vioif_ctrl_cmdspec *specs, int nspecs) +{ + struct virtio_softc *vsc = sc->sc_virtio; + int i; - mutex_enter(&sc->sc_ctrl_wait_lock); - while (sc->sc_ctrl_inuse != FREE) - cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); - sc->sc_ctrl_inuse = INUSE; - mutex_exit(&sc->sc_ctrl_wait_lock); + for (i = 0; i < nspecs; i++) { + bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); + } +} + +static int +vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, + struct vioif_ctrl_cmdspec *specs, int nspecs) +{ + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + struct virtqueue *vq = ctrlq->ctrlq_vq; + struct virtio_softc *vsc = sc->sc_virtio; + int i, r, slot; - sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX; - sc->sc_ctrl_cmd->command = cmd; - sc->sc_ctrl_rx->onoff = onoff; + ctrlq->ctrlq_cmd->class = class; + ctrlq->ctrlq_cmd->command = cmd; - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, + bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, - 0, sizeof(struct virtio_net_ctrl_rx), - BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, + for (i = 0; i < nspecs; i++) { + bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, + 0, specs[i].bufsize, + BUS_DMASYNC_PREWRITE); + } + bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); r = virtio_enqueue_prep(vsc, vq, &slot); if (r != 0) panic("%s: control vq busy!?", device_xname(sc->sc_dev)); - r = virtio_enqueue_reserve(vsc, vq, slot, 3); + r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); if (r != 0) panic("%s: control vq busy!?", device_xname(sc->sc_dev)); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); + virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); + for (i = 0; i < nspecs; i++) { + virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); + } + virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); virtio_enqueue_commit(vsc, vq, slot, true); /* wait for done */ - mutex_enter(&sc->sc_ctrl_wait_lock); - while (sc->sc_ctrl_inuse != DONE) - cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); - mutex_exit(&sc->sc_ctrl_wait_lock); + mutex_enter(&ctrlq->ctrlq_wait_lock); + while (ctrlq->ctrlq_inuse != DONE) + cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); + mutex_exit(&ctrlq->ctrlq_wait_lock); /* already dequeueued */ - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, + bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 0, - sizeof(struct virtio_net_ctrl_rx), - BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, + for (i = 0; i < nspecs; i++) { + bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, + specs[i].bufsize, + BUS_DMASYNC_POSTWRITE); + } + bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); - if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) + if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) r = 0; else { printf("%s: failed setting rx mode\n", @@ -1275,11 +1759,30 @@ vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) r = EIO; } - mutex_enter(&sc->sc_ctrl_wait_lock); - sc->sc_ctrl_inuse = FREE; - cv_signal(&sc->sc_ctrl_wait); - mutex_exit(&sc->sc_ctrl_wait_lock); + return r; +} + +static int +vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) +{ + struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; + struct vioif_ctrl_cmdspec specs[1]; + int r; + + if (!sc->sc_has_ctrl) + return ENOTSUP; + + vioif_ctrl_acquire(sc); + rx->onoff = onoff; + specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; + specs[0].buf = rx; + specs[0].bufsize = sizeof(*rx); + + r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, + specs, __arraycount(specs)); + + vioif_ctrl_release(sc); return r; } @@ -1307,109 +1810,71 @@ vioif_set_allmulti(struct vioif_softc *sc, bool onoff) static int vioif_set_rx_filter(struct vioif_softc *sc) { - /* filter already set in sc_ctrl_mac_tbl */ - struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; - int r, slot; + /* filter already set in ctrlq->ctrlq_mac_tbl */ + struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; + struct vioif_ctrl_cmdspec specs[2]; + int nspecs = __arraycount(specs); + int r; + + mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; + mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; if (!sc->sc_has_ctrl) return ENOTSUP; - mutex_enter(&sc->sc_ctrl_wait_lock); - while (sc->sc_ctrl_inuse != FREE) - cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); - sc->sc_ctrl_inuse = INUSE; - mutex_exit(&sc->sc_ctrl_wait_lock); + vioif_ctrl_acquire(sc); - sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC; - sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET; + specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; + specs[0].buf = mac_tbl_uc; + specs[0].bufsize = sizeof(*mac_tbl_uc) + + (ETHER_ADDR_LEN * mac_tbl_uc->nentries); - r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, - sc->sc_ctrl_mac_tbl_uc, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), - NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); - if (r) { - printf("%s: control command dmamap load failed, " - "error code %d\n", device_xname(sc->sc_dev), r); - goto out; - } - r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, - sc->sc_ctrl_mac_tbl_mc, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), - NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); - if (r) { - printf("%s: control command dmamap load failed, " - "error code %d\n", device_xname(sc->sc_dev), r); - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); + specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; + specs[1].buf = mac_tbl_mc; + specs[1].bufsize = sizeof(*mac_tbl_mc) + + (ETHER_ADDR_LEN * mac_tbl_mc->nentries); + + r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); + if (r != 0) goto out; - } - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, - 0, sizeof(struct virtio_net_ctrl_cmd), - BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), - BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), - BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, - 0, sizeof(struct virtio_net_ctrl_status), - BUS_DMASYNC_PREREAD); + r = vioif_ctrl_send_command(sc, + VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, + specs, nspecs); - r = virtio_enqueue_prep(vsc, vq, &slot); - if (r != 0) - panic("%s: control vq busy!?", device_xname(sc->sc_dev)); - r = virtio_enqueue_reserve(vsc, vq, slot, 4); - if (r != 0) - panic("%s: control vq busy!?", device_xname(sc->sc_dev)); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true); - virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); - virtio_enqueue_commit(vsc, vq, slot, true); + vioif_ctrl_unload_cmdspec(sc, specs, nspecs); - /* wait for done */ - mutex_enter(&sc->sc_ctrl_wait_lock); - while (sc->sc_ctrl_inuse != DONE) - cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); - mutex_exit(&sc->sc_ctrl_wait_lock); - /* already dequeueued */ +out: + vioif_ctrl_release(sc); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, - sizeof(struct virtio_net_ctrl_cmd), - BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), - BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, - (sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), - BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, - sizeof(struct virtio_net_ctrl_status), - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); - bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap); + return r; +} - if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) - r = 0; - else { - printf("%s: failed setting rx filter\n", - device_xname(sc->sc_dev)); - r = EIO; - } +static int +vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) +{ + struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; + struct vioif_ctrl_cmdspec specs[1]; + int r; -out: - mutex_enter(&sc->sc_ctrl_wait_lock); - sc->sc_ctrl_inuse = FREE; - cv_signal(&sc->sc_ctrl_wait); - mutex_exit(&sc->sc_ctrl_wait_lock); + if (!sc->sc_has_ctrl) + return ENOTSUP; + + if (nvq_pairs <= 1) + return EINVAL; + + vioif_ctrl_acquire(sc); + + mq->virtqueue_pairs = nvq_pairs; + specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; + specs[0].buf = mq; + specs[0].bufsize = sizeof(*mq); + + r = vioif_ctrl_send_command(sc, + VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, + specs, __arraycount(specs)); + + vioif_ctrl_release(sc); return r; } @@ -1419,7 +1884,7 @@ static int vioif_ctrl_vq_done(struct virtqueue *vq) { struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct vioif_ctrlqueue *ctrlq = vq->vq_done_ctx; int r, slot; r = virtio_dequeue(vsc, vq, &slot, NULL); @@ -1427,10 +1892,10 @@ vioif_ctrl_vq_done(struct virtqueue *vq) return 0; virtio_dequeue_commit(vsc, vq, slot); - mutex_enter(&sc->sc_ctrl_wait_lock); - sc->sc_ctrl_inuse = DONE; - cv_signal(&sc->sc_ctrl_wait); - mutex_exit(&sc->sc_ctrl_wait_lock); + mutex_enter(&ctrlq->ctrlq_wait_lock); + ctrlq->ctrlq_inuse = DONE; + cv_signal(&ctrlq->ctrlq_wait); + mutex_exit(&ctrlq->ctrlq_wait_lock); return 1; } @@ -1450,6 +1915,7 @@ vioif_rx_filter(struct vioif_softc *sc) struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct ether_multi *enm; struct ether_multistep step; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; int nentries; int promisc = 0, allmulti = 0, rxfilter = 0; int r; @@ -1477,7 +1943,7 @@ vioif_rx_filter(struct vioif_softc *sc) allmulti = 1; goto set_unlock; } - memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries], + memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], enm->enm_addrlo, ETHER_ADDR_LEN); ETHER_NEXT_MULTI(step, enm); } @@ -1488,8 +1954,8 @@ set_unlock: set: if (rxfilter) { - sc->sc_ctrl_mac_tbl_uc->nentries = 0; - sc->sc_ctrl_mac_tbl_mc->nentries = nentries; + ctrlq->ctrlq_mac_tbl_uc->nentries = 0; + ctrlq->ctrlq_mac_tbl_mc->nentries = nentries; r = vioif_set_rx_filter(sc); if (r != 0) { rxfilter = 0; @@ -1497,8 +1963,8 @@ set: } } else { /* remove rx filter */ - sc->sc_ctrl_mac_tbl_uc->nentries = 0; - sc->sc_ctrl_mac_tbl_mc->nentries = 0; + ctrlq->ctrlq_mac_tbl_uc->nentries = 0; + ctrlq->ctrlq_mac_tbl_mc->nentries = 0; r = vioif_set_rx_filter(sc); /* what to do on failure? */ } @@ -1541,13 +2007,13 @@ static void vioif_update_link_status(struct vioif_softc *sc) { struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_txqueue *txq; bool active, changed; - int link; + int link, i; active = vioif_is_link_up(sc); changed = false; - VIOIF_TX_LOCK(sc); if (active) { if (!sc->sc_link_active) changed = true; @@ -1561,10 +2027,18 @@ vioif_update_link_status(struct vioif_softc *sc) link = LINK_STATE_DOWN; sc->sc_link_active = false; } - VIOIF_TX_UNLOCK(sc); - if (changed) + if (changed) { + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + txq = &sc->sc_txq[i]; + + VIOIF_TXQ_LOCK(txq); + txq->txq_link_active = sc->sc_link_active; + VIOIF_TXQ_UNLOCK(txq); + } + if_link_state_change(ifp, link); + } } static int diff --git a/sys/dev/pci/virtio.c b/sys/dev/pci/virtio.c index d03cffa1b91..5955f497ac0 100644 --- a/sys/dev/pci/virtio.c +++ b/sys/dev/pci/virtio.c @@ -235,6 +235,54 @@ vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, ops); } +static void +virtio_vq_soft_intr(void *arg) +{ + struct virtqueue *vq = arg; + + KASSERT(vq->vq_intrhand != NULL); + + (vq->vq_intrhand)(vq); +} + +static int +virtio_vq_softint_establish(struct virtio_softc *sc) +{ + struct virtqueue *vq; + int qid; + u_int flags; + + flags = SOFTINT_NET; + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) + flags |= SOFTINT_MPSAFE; + + for (qid = 0; qid < sc->sc_nvqs; qid++) { + vq = &sc->sc_vqs[qid]; + vq->vq_soft_ih = + softint_establish(flags, virtio_vq_soft_intr, vq); + if (vq->vq_soft_ih == NULL) + return -1; + } + + return 0; +} + +static void +virtio_vq_softint_disestablish(struct virtio_softc *sc) +{ + struct virtqueue *vq; + int qid; + + for (qid = 0; qid < sc->sc_nvqs; qid++) { + vq = &sc->sc_vqs[qid]; + if (vq->vq_soft_ih == NULL) + continue; + + softint_disestablish(vq->vq_soft_ih); + vq->vq_soft_ih = NULL; + } +} + /* * Can be used as sc_intrhand. */ @@ -242,6 +290,26 @@ vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, * Scan vq, bus_dmamap_sync for the vqs (not for the payload), * and calls (*vq_done)() if some entries are consumed. */ +static int +virtio_vq_intr_common(struct virtqueue *vq) +{ + struct virtio_softc *sc = vq->vq_owner; + int r = 0; + + if (vq->vq_queued) { + vq->vq_queued = 0; + vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); + } + vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); + membar_consumer(); + if (vq->vq_used_idx != vq->vq_used->idx) { + if (vq->vq_done) + r |= (vq->vq_done)(vq); + } + + return r; +} + int virtio_vq_intr(struct virtio_softc *sc) { @@ -250,21 +318,19 @@ virtio_vq_intr(struct virtio_softc *sc) for (i = 0; i < sc->sc_nvqs; i++) { vq = &sc->sc_vqs[i]; - if (vq->vq_queued) { - vq->vq_queued = 0; - vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); - } - vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); - membar_consumer(); - if (vq->vq_used_idx != vq->vq_used->idx) { - if (vq->vq_done) - r |= (vq->vq_done)(vq); - } + r |= virtio_vq_intr_common(vq); } return r; } +static int +virtio_vq_mq_intr(struct virtqueue *vq) +{ + + return virtio_vq_intr_common(vq); +} + /* * Start/stop vq interrupt. No guarantee. */ @@ -409,6 +475,7 @@ virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, /* remember addresses and offsets for later use */ vq->vq_owner = sc; + vq->vq_intrhand = virtio_vq_mq_intr; vq->vq_num = vq_size; vq->vq_index = index; vq->vq_desc = vq->vq_vaddr; @@ -844,6 +911,16 @@ virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, aprint_naive("\n"); } +void +virtio_child_attach_set_vqs(struct virtio_softc *sc, + struct virtqueue *vqs, int nvq_pairs) +{ + if (nvq_pairs > 1) + sc->sc_child_mq = true; + + sc->sc_vqs = vqs; +} + int virtio_child_attach_finish(struct virtio_softc *sc) { @@ -868,12 +945,26 @@ virtio_child_attach_finish(struct virtio_softc *sc) "failed to establish soft interrupt\n"); goto fail; } + + if (sc->sc_child_mq) { + r = virtio_vq_softint_establish(sc); + aprint_error_dev(sc->sc_dev, + "failed to establish softint interrupt\n"); + goto fail; + } } virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); return 0; fail: + if (sc->sc_soft_ih) { + softint_disestablish(sc->sc_soft_ih); + sc->sc_soft_ih = NULL; + } + + virtio_vq_softint_disestablish(sc); + virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); return 1; } diff --git a/sys/dev/pci/virtio_pci.c b/sys/dev/pci/virtio_pci.c index 65c5222b774..bb972997be2 100644 --- a/sys/dev/pci/virtio_pci.c +++ b/sys/dev/pci/virtio_pci.c @@ -32,6 +32,7 @@ __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.5 2018/06/06 16:11:36 jakllsch Exp #include #include #include +#include #include @@ -79,6 +80,7 @@ static void virtio_pci_free_interrupts(struct virtio_softc *); static int virtio_pci_intr(void *arg); static int virtio_pci_msix_queue_intr(void *); +static int virtio_pci_msix_vq_intr(void *); static int virtio_pci_msix_config_intr(void *); static int virtio_pci_setup_msix_vectors(struct virtio_softc *); static int virtio_pci_setup_msix_interrupts(struct virtio_softc *, @@ -428,7 +430,7 @@ virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr) if (psc->sc_ihs_num > 1) { int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; - if (false) /* (for per-vq vectors) */ + if (sc->sc_child_mq) vec += idx; bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); @@ -488,6 +490,9 @@ virtio_pci_setup_msix_vectors(struct virtio_softc *sc) offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; + if (sc->sc_child_mq) + vector += qid; + bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector); ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset); aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", @@ -507,29 +512,57 @@ virtio_pci_setup_msix_interrupts(struct virtio_softc *sc, device_t self = sc->sc_dev; pci_chipset_tag_t pc = pa->pa_pc; char intrbuf[PCI_INTRSTR_LEN]; + char intr_xname[INTRDEVNAMEBUF]; char const *intrstr; - int idx; + int idx, qid, n; idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); + snprintf(intr_xname, sizeof(intr_xname), "%s config", + device_xname(sc->sc_dev)); + psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], - sc->sc_ipl, virtio_pci_msix_config_intr, sc, device_xname(sc->sc_dev)); + sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); if (psc->sc_ihs[idx] == NULL) { aprint_error_dev(self, "couldn't establish MSI-X for config\n"); goto error; } idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; - if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) - pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); - - psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], - sc->sc_ipl, virtio_pci_msix_queue_intr, sc, device_xname(sc->sc_dev)); - if (psc->sc_ihs[idx] == NULL) { - aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); - goto error; + if (sc->sc_child_mq) { + for (qid = 0; qid < sc->sc_nvqs; qid++) { + n = idx + qid; + + snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", + device_xname(sc->sc_dev), qid); + + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) { + pci_intr_setattr(pc, &psc->sc_ihp[n], + PCI_INTR_MPSAFE, true); + } + + psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n], + sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid], + intr_xname); + if (psc->sc_ihs[n] == NULL) { + aprint_error_dev(self, "couldn't establish MSI-X for a vq\n"); + goto error; + } + } + } else { + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) + pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); + + snprintf(intr_xname, sizeof(intr_xname), "%s queues", + device_xname(sc->sc_dev)); + psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], + sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname); + if (psc->sc_ihs[idx] == NULL) { + aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); + goto error; + } } if (virtio_pci_setup_msix_vectors(sc) != 0) { @@ -541,8 +574,38 @@ virtio_pci_setup_msix_interrupts(struct virtio_softc *sc, intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); aprint_normal_dev(self, "config interrupting at %s\n", intrstr); idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; - intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); - aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); + if (sc->sc_child_mq) { + kcpuset_t *affinity; + int affinity_to, r; + + kcpuset_create(&affinity, false); + + for (qid = 0; qid < sc->sc_nvqs; qid++) { + n = idx + qid; + affinity_to = (qid / 2) % ncpu; + + intrstr = pci_intr_string(pc, psc->sc_ihp[n], + intrbuf, sizeof(intrbuf)); + + kcpuset_zero(affinity); + kcpuset_set(affinity, affinity_to); + r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL); + if (r == 0) { + aprint_normal_dev(self, + "for vq #%d interrupting at %s affinity to %u\n", + qid, intrstr, affinity_to); + } else { + aprint_normal_dev(self, + "for vq #%d interrupting at %s\n", + qid, intrstr); + } + } + + kcpuset_destroy(affinity); + } else { + intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); + aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); + } return 0; @@ -551,8 +614,18 @@ error: if (psc->sc_ihs[idx] != NULL) pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; - if (psc->sc_ihs[idx] != NULL) - pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); + if (sc->sc_child_mq) { + for (qid = 0; qid < sc->sc_nvqs; qid++) { + n = idx + qid; + if (psc->sc_ihs[n] == NULL) + continue; + pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]); + } + + } else { + if (psc->sc_ihs[idx] != NULL) + pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); + } return -1; } @@ -604,8 +677,14 @@ virtio_pci_setup_interrupts(struct virtio_softc *sc) counts[PCI_INTR_TYPE_INTX] = 1; } else { /* Try MSI-X first and INTx second */ + if (sc->sc_child_mq && + sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) { + nmsix = 2; + sc->sc_child_mq = false; + } + max_type = PCI_INTR_TYPE_MSIX; - counts[PCI_INTR_TYPE_MSIX] = 2; + counts[PCI_INTR_TYPE_MSIX] = nmsix; counts[PCI_INTR_TYPE_MSI] = 0; counts[PCI_INTR_TYPE_INTX] = 1; } @@ -618,13 +697,13 @@ retry: } if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { - psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 2, + psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * nmsix, KM_SLEEP); error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa); if (error != 0) { - kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 2); - pci_intr_release(pc, psc->sc_ihp, 2); + kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); + pci_intr_release(pc, psc->sc_ihp, nmsix); /* Retry INTx */ max_type = PCI_INTR_TYPE_INTX; @@ -632,7 +711,7 @@ retry: goto retry; } - psc->sc_ihs_num = 2; + psc->sc_ihs_num = nmsix; psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1, @@ -718,6 +797,22 @@ virtio_pci_msix_queue_intr(void *arg) return r; } +static int +virtio_pci_msix_vq_intr(void *arg) +{ + struct virtqueue *vq = arg; + int r = 0; + + if (vq->vq_intrhand != NULL) { + if (vq->vq_soft_ih) + softint_schedule(vq->vq_soft_ih); + else + r |= vq->vq_intrhand(vq); + } + + return r; +} + static int virtio_pci_msix_config_intr(void *arg) { diff --git a/sys/dev/pci/virtiovar.h b/sys/dev/pci/virtiovar.h index 7c9a9a3689b..4c05b6af293 100644 --- a/sys/dev/pci/virtiovar.h +++ b/sys/dev/pci/virtiovar.h @@ -116,6 +116,9 @@ struct virtqueue { /* interrupt handler */ int (*vq_done)(struct virtqueue*); + void *vq_done_ctx; + void *vq_soft_ih; + int (*vq_intrhand)(struct virtqueue*); }; struct virtio_attach_args { @@ -161,6 +164,7 @@ struct virtio_softc { int sc_childdevid; device_t sc_child; /* set by child */ + bool sc_child_mq; virtio_callback sc_config_change; /* set by child */ virtio_callback sc_intrhand; /* set by child */ }; @@ -196,6 +200,8 @@ void virtio_child_attach_start(struct virtio_softc *, device_t, int, struct virtqueue *, virtio_callback, virtio_callback, int, int, const char *); +void virtio_child_attach_set_vqs(struct virtio_softc *, + struct virtqueue *, int); int virtio_child_attach_finish(struct virtio_softc *); void virtio_child_attach_failed(struct virtio_softc *); void virtio_child_detach(struct virtio_softc *);