37#include <sys/kernel.h>
39#include <sys/malloc.h>
42#include <sys/sglist.h>
46#include <vm/vm_page.h>
66#define SGLIST_SAVE(sg, sgsave) do { \
67 (sgsave).sg_nseg = (sg)->sg_nseg; \
68 if ((sgsave).sg_nseg > 0) \
69 (sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \
71 (sgsave).ss_len = 0; \
74#define SGLIST_RESTORE(sg, sgsave) do { \
75 (sg)->sg_nseg = (sgsave).sg_nseg; \
76 if ((sgsave).sg_nseg > 0) \
77 (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \
87 vm_paddr_t paddr,
size_t len)
89 struct sglist_seg *ss;
92 if (ss->ss_paddr + ss->ss_len == paddr)
95 if (sg->sg_nseg == sg->sg_maxseg)
114 struct sglist_seg *ss;
115 vm_offset_t vaddr, offset;
126 vaddr = (vm_offset_t)
buf;
127 offset = vaddr & PAGE_MASK;
129 paddr = pmap_extract(pmap, vaddr);
131 paddr = pmap_kextract(vaddr);
132 seglen = MIN(len, PAGE_SIZE - offset);
133 if (sg->sg_nseg == 0) {
135 ss->ss_paddr = paddr;
139 ss = &sg->sg_segs[sg->sg_nseg - 1];
150 seglen = MIN(len, PAGE_SIZE);
152 paddr = pmap_extract(pmap, vaddr);
154 paddr = pmap_kextract(vaddr);
174 vm_offset_t vaddr, vendaddr;
175 vm_paddr_t lastaddr, paddr;
181 vaddr = trunc_page((vm_offset_t)
buf);
182 vendaddr = (vm_offset_t)
buf + len;
184 lastaddr = pmap_kextract(vaddr);
186 while (vaddr < vendaddr) {
187 paddr = pmap_kextract(vaddr);
188 if (lastaddr + PAGE_SIZE != paddr)
203 vm_paddr_t lastaddr, paddr;
211 lastaddr = VM_PAGE_TO_PHYS(m[0]);
212 for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) {
213 paddr = VM_PAGE_TO_PHYS(m[i]);
214 if (lastaddr + PAGE_SIZE != paddr)
228 vm_paddr_t nextaddr, paddr;
229 size_t seglen, segoff;
230 int i, nsegs, pglen, pgoff;
236 if (m->m_epg_hdrlen != 0) {
237 if (off >= m->m_epg_hdrlen) {
238 off -= m->m_epg_hdrlen;
240 seglen = m->m_epg_hdrlen - off;
242 seglen = MIN(seglen, len);
250 pgoff = m->m_epg_1st_off;
251 for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
252 pglen = m_epg_pagelen(m, i, pgoff);
258 seglen = pglen - off;
259 segoff = pgoff + off;
261 seglen = MIN(seglen, len);
263 paddr = m->m_epg_pa[i] + segoff;
264 if (paddr != nextaddr)
266 nextaddr = paddr + seglen;
270 seglen = MIN(len, m->m_epg_trllen - off);
274 KASSERT(len == 0, (
"len != 0"));
288 sg =
malloc(
sizeof(
struct sglist) + nsegs *
sizeof(
struct sglist_seg),
292 sglist_init(sg, nsegs, (
struct sglist_seg *)(sg + 1));
306 if (refcount_release(&sg->sg_refs))
321 if (sg->sg_maxseg == 0)
341 if ((bp->bio_flags & BIO_UNMAPPED) == 0)
345 bp->bio_ma_offset, bp->bio_bcount);
356 struct sglist_seg *ss;
360 if (sg->sg_maxseg == 0)
365 if (sg->sg_nseg == 0) {
366 sg->sg_segs[0].ss_paddr = paddr;
367 sg->sg_segs[0].
ss_len = len;
371 ss = &sg->sg_segs[sg->sg_nseg - 1];
387 size_t seglen, segoff;
389 int error, i, pglen, pgoff;
394 if (m->m_epg_hdrlen != 0) {
395 if (off >= m->m_epg_hdrlen) {
396 off -= m->m_epg_hdrlen;
398 seglen = m->m_epg_hdrlen - off;
400 seglen = MIN(seglen, len);
404 &m->m_epg_hdr[segoff], seglen);
407 pgoff = m->m_epg_1st_off;
408 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) {
409 pglen = m_epg_pagelen(m, i, pgoff);
415 seglen = pglen - off;
416 segoff = pgoff + off;
418 seglen = MIN(seglen, len);
420 paddr = m->m_epg_pa[i] + segoff;
424 if (error == 0 && len > 0) {
425 seglen = MIN(len, m->m_epg_trllen - off);
428 &m->m_epg_trail[off], seglen);
431 KASSERT(len == 0, (
"len != 0"));
447 if (sg->sg_maxseg == 0)
452 for (m = m0; m != NULL; m = m->m_next) {
454 if ((m->m_flags & M_EXTPG) != 0)
456 mtod(m, vm_offset_t), m->m_len);
477 if ((m->m_flags & M_EXTPG) != 0)
479 mtod(m, vm_offset_t), m->m_len));
494 struct sglist_seg *ss;
499 if (sg->sg_maxseg == 0)
506 if (sg->sg_nseg == 0) {
507 seglen = min(PAGE_SIZE - pgoff, len);
508 sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff;
509 sg->sg_segs[0].ss_len = seglen;
515 ss = &sg->sg_segs[sg->sg_nseg - 1];
516 for (; len > 0; i++, len -= seglen) {
517 seglen = min(PAGE_SIZE - pgoff, len);
518 paddr = VM_PAGE_TO_PHYS(m[i]) + pgoff;
540 if (sg->sg_maxseg == 0)
544 vmspace_pmap(td->td_proc->p_vmspace), NULL);
560 struct sglist_seg *ss;
564 if (sg->sg_maxseg == 0 || length == 0)
568 ss = &sg->sg_segs[sg->sg_nseg - 1];
569 for (i = 0; i < source->sg_nseg; i++) {
570 if (offset >= source->sg_segs[i].ss_len) {
571 offset -= source->sg_segs[i].ss_len;
574 seglen = source->sg_segs[i].ss_len - offset;
578 source->sg_segs[i].ss_paddr + offset, seglen);
603 size_t resid, minlen;
607 if (sg->sg_maxseg == 0)
610 resid = uio->uio_resid;
613 if (uio->uio_segflg == UIO_USERSPACE) {
614 KASSERT(uio->uio_td != NULL,
615 (
"sglist_append_uio: USERSPACE but no thread"));
616 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
622 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
627 minlen = MIN(resid, iov[i].iov_len);
654 if (sg->sg_maxseg == 0)
657 if (uio->uio_segflg == UIO_USERSPACE) {
658 KASSERT(uio->uio_td != NULL,
659 (
"sglist_consume_uio: USERSPACE but no thread"));
660 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
665 while (resid > 0 && uio->uio_resid) {
681 iov->iov_base = (
char *)iov->iov_base + done;
682 iov->iov_len -= done;
683 uio->uio_resid -= done;
684 uio->uio_offset += done;
729 new->sg_nseg = sg->sg_nseg;
730 bcopy(sg->sg_segs, new->sg_segs,
sizeof(
struct sglist_seg) *
746 for (i = 0; i < sg->sg_nseg; i++)
747 space += sg->sg_segs[i].ss_len;
769sglist_split(
struct sglist *original,
struct sglist **head,
size_t length,
776 if (original->sg_refs > 1)
783 for (i = 0; i < original->sg_nseg; i++) {
784 space += original->sg_segs[i].ss_len;
786 if (space >= length) {
793 split = space - length;
809 if (sg->sg_maxseg <
count)
811 if (sg->sg_nseg != 0)
816 bcopy(original->sg_segs, sg->sg_segs,
count *
817 sizeof(
struct sglist_seg));
828 sg->sg_segs[
count].ss_len -= split;
829 original->sg_segs[
count].ss_paddr =
830 sg->sg_segs[
count].ss_paddr + split;
831 original->sg_segs[
count].ss_len = split;
835 original->sg_nseg -=
count;
836 bcopy(original->sg_segs +
count, original->sg_segs,
count *
837 sizeof(
struct sglist_seg));
849 struct sglist_seg *flast, *sfirst;
853 if (second->sg_nseg == 0)
861 flast = &first->sg_segs[first->sg_nseg - 1];
862 sfirst = &second->sg_segs[0];
863 if (first->sg_nseg != 0 &&
864 flast->ss_paddr + flast->ss_len == sfirst->ss_paddr)
868 if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg)
873 flast->ss_len += sfirst->ss_len;
876 bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append,
877 (second->sg_nseg - append) *
sizeof(
struct sglist_seg));
878 first->sg_nseg += second->sg_nseg - append;
879 sglist_reset(second);
900sglist_slice(
struct sglist *original,
struct sglist **slice,
size_t offset,
901 size_t length,
int mflags)
904 size_t space, end, foffs, loffs;
912 end = offset + length;
917 for (i = 0; i < original->sg_nseg; i++) {
918 space += original->sg_segs[i].ss_len;
919 if (space > offset) {
927 foffs = offset - (space -
928 original->sg_segs[i].ss_len);
929 CTR1(KTR_DEV,
"sglist_slice: foffs = %08lx",
941 CTR1(KTR_DEV,
"sglist_slice: loffs = %08lx",
952 if (*slice == NULL) {
959 if (sg->sg_maxseg <
count)
961 if (sg->sg_nseg != 0)
969 bcopy(original->sg_segs + fseg, sg->sg_segs,
970 count *
sizeof(
struct sglist_seg));
975 sg->sg_segs[0].ss_paddr += foffs;
976 sg->sg_segs[0].ss_len -= foffs;
977 CTR2(KTR_DEV,
"sglist_slice seg[0]: %08lx:%08lx",
978 (
long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len);
981 sg->sg_segs[
count - 1].ss_len -= loffs;
982 CTR2(KTR_DEV,
"sglist_slice seg[%d]: len %08x",
count - 1,
983 sg->sg_segs[
count - 1].ss_len);
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
int sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len)
void sglist_free(struct sglist *sg)
int sglist_slice(struct sglist *original, struct sglist **slice, size_t offset, size_t length, int mflags)
int sglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset, size_t length)
int sglist_append_bio(struct sglist *sg, struct bio *bp)
int sglist_append(struct sglist *sg, void *buf, size_t len)
int sglist_split(struct sglist *original, struct sglist **head, size_t length, int mflags)
int sglist_count(void *buf, size_t len)
int sglist_join(struct sglist *first, struct sglist *second)
#define SGLIST_SAVE(sg, sgsave)
#define SGLIST_RESTORE(sg, sgsave)
int sglist_append_uio(struct sglist *sg, struct uio *uio)
int sglist_append_vmpages(struct sglist *sg, vm_page_t *m, size_t pgoff, size_t len)
int sglist_append_mbuf(struct sglist *sg, struct mbuf *m0)
static MALLOC_DEFINE(M_SGLIST, "sglist", "scatter/gather lists")
int sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid)
int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len)
static __inline int _sglist_append_range(struct sglist *sg, struct sglist_seg **ssp, vm_paddr_t paddr, size_t len)
struct sglist * sglist_clone(struct sglist *sg, int mflags)
size_t sglist_length(struct sglist *sg)
int sglist_append_mbuf_epg(struct sglist *sg, struct mbuf *m, size_t off, size_t len)
int sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td)
struct sglist * sglist_build(void *buf, size_t len, int mflags)
struct sglist * sglist_alloc(int nsegs, int mflags)
int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len)
static __inline int _sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap, size_t *donep)
int sglist_append_single_mbuf(struct sglist *sg, struct mbuf *m)