Xeffy/gouring
1
0
Fork 0
mirror of https://github.com/ii64/gouring.git synced 2025-04-07 22:33:22 +02:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

...

No commits in common. "v0.2.3" and "v0.4" have entirely different histories.
v0.2.3 ... v0.4

38 changed files with 4239 additions and 1159 deletions

4
.gitignore vendored Normal file
View file

@ -0,0 +1,4 @@
.vscode/
*.cpu
*.mem
*.test

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2021 Ii64人
Copyright (c) 2022 gouring Author
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

5
Makefile Normal file
View file

@ -0,0 +1,5 @@
test:
go test -c -gcflags=all=-d=checkptr .
./gouring.test
clean:
rm -rf *.test

98
README.md
View file

@ -1,79 +1,49 @@
# gouring
[![License: MIT][1]](LICENSE)
[![Go Reference][2]][3]
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
[![Go Reference](https://pkg.go.dev/badge/github.com/ii64/gouring.svg)](https://pkg.go.dev/github.com/ii64/gouring)
Low-level io uring library
```
```bash
go get github.com/ii64/gouring
```
## Example
```go
import "github.com/ii64/gouring"
import "github.com/ii64/gouring/queue"
// setup
h, err := gouring.New(256, 0)
if err != nil { /*...*/ }
defer h.Close()
// io_uring_setup
ring, err := gouring.New(256, nil) // default io uring setup param
if err != nil {
// ...
}
defer ring.Close() // munmap shared memory, cleanup
var (
ret int
err error
)
sqe := h.GetSQE()
b := []byte("io_uring!\n")
PrepWrite(sqe, 1, &b[0], len(b), 0)
// io_uring_register
ret, err = ring.Register(gouring.IORING_REGISTER_BUFFERS, addr, nrArg)
submitted, err := h.SubmitAndWait(1)
if err != nil { /*...*/ }
println(submitted) // 1
// io_uring_enter
ret, err = ring.Enter(toSubmit, minComplete, gouring.IORING_ENTER_GETEVENTS, nil)
var cqe *gouring.IoUringCqe
err = h.WaitCqe(&cqe)
if err != nil { /*...*/ } // check also EINTR
// setup param
params := ring.Params()
// ring fd
fd := ring.Fd()
// Submission Queue
sq := ring.SQ()
// Completion Queue
cq := ring.CQ()
/* Using queue package */
q := queue.New(ring)
go func() {
q.Run(func(cqe *gouring.CQEntry) {
// cqe processing
_ = cqe.UserData
_ = cqe.Res
_ = cqe.Flags
})
}()
// buffer
data := []byte("print on stdout\n")
// get sqe
sqe := q.GetSQEntry()
sqe.UserData = 0 // identifier / event id
sqe.Opcode = gouring.IORING_OP_WRITE // op write
sqe.Fd = int32(syscall.Stdout) // fd 1
sqe.Len = uint32(len(data)) // buffer size
sqe.SetOffset(0) // fd offset
sqe.SetAddr(&data[0]) // buffer addr
// submit sqe
submitted, err := q.Submit()
if err != nil {
// ...
}
_ = cqe.UserData
_ = cqe.Res
_ = cqe.Flags
```
### Referece
[github.com/iceber/iouring-go](https://github.com/iceber/iouring-go)
## Graph
| SQPOLL | non-SQPOLL |
| ------ | ---------- |
| ![sqpoll_fig][sqpoll_fig] | ![nonsqpoll_fig][nonsqpoll_fig] |
### Reference
https://github.com/axboe/liburing
[1]: https://img.shields.io/badge/License-MIT-yellow.svg
[2]: https://pkg.go.dev/badge/github.com/ii64/gouring.svg
[3]: https://pkg.go.dev/github.com/ii64/gouring
[sqpoll_fig]: assets/sqpoll.svg
[nonsqpoll_fig]: assets/nonsqpoll.svg

533
assets/nonsqpoll.svg Normal file
View file

@ -0,0 +1,533 @@
<svg width="552pt" height="1038pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1.3 1.3) rotate(0) translate(4 1034)">
<title>perf</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-1034 547.5,-1034 547.5,4 -4,4"></polygon>
<!-- N1 -->
<g id="node1" class="node">
<title>N1</title>
<g id="a_node1"><a xlink:title="main.main (111.14s)">
<polygon fill="#edd5d5" stroke="#b20000" points="248,-943 118,-943 118,-875 248,-875 248,-943"></polygon>
<text text-anchor="middle" x="183" y="-927.8" font-family="Times,serif" font-size="14.00">main</text>
<text text-anchor="middle" x="183" y="-912.8" font-family="Times,serif" font-size="14.00">main</text>
<text text-anchor="middle" x="183" y="-897.8" font-family="Times,serif" font-size="14.00">9.73s (8.75%)</text>
<text text-anchor="middle" x="183" y="-882.8" font-family="Times,serif" font-size="14.00">of 111.14s (99.92%)</text>
</a>
</g>
</g>
<!-- N10 -->
<g id="node10" class="node">
<title>N10</title>
<g id="a_node10"><a xlink:title="github.com/ii64/gouring.(*IoUring).GetSqe (4.08s)">
<polygon fill="#edebe8" stroke="#b2a691" points="118.5,-809 35.5,-809 35.5,-746 118.5,-746 118.5,-809"></polygon>
<text text-anchor="middle" x="77" y="-797" font-family="Times,serif" font-size="10.00">gouring</text>
<text text-anchor="middle" x="77" y="-786" font-family="Times,serif" font-size="10.00">(*IoUring)</text>
<text text-anchor="middle" x="77" y="-775" font-family="Times,serif" font-size="10.00">GetSqe</text>
<text text-anchor="middle" x="77" y="-764" font-family="Times,serif" font-size="10.00">0.39s (0.35%)</text>
<text text-anchor="middle" x="77" y="-753" font-family="Times,serif" font-size="10.00">of 4.08s (3.67%)</text>
</a>
</g>
</g>
<!-- N1&#45;&gt;N10 -->
<g id="edge14" class="edge">
<title>N1-&gt;N10</title>
<g id="a_edge14"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).GetSqe (4.08s)">
<path fill="none" stroke="#b2a691" d="M150.66,-874.77C145.31,-868.95 139.91,-862.88 135,-857 124.58,-844.52 113.82,-830.36 104.44,-817.54"></path>
<polygon fill="#b2a691" stroke="#b2a691" points="107.19,-815.38 98.49,-809.34 101.53,-819.49 107.19,-815.38"></polygon>
</a>
</g>
<g id="a_edge14-label"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).GetSqe (4.08s)">
<text text-anchor="middle" x="157" y="-845.8" font-family="Times,serif" font-size="14.00"> 4.08s</text>
<text text-anchor="middle" x="157" y="-830.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N14 -->
<g id="node14" class="node">
<title>N14</title>
<g id="a_node14"><a xlink:title="github.com/ii64/gouring.(*IoUring).Submit (81.67s)">
<polygon fill="#edd7d5" stroke="#b20f00" points="226,-806.5 140,-806.5 140,-748.5 226,-748.5 226,-806.5"></polygon>
<text text-anchor="middle" x="183" y="-795.3" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="183" y="-785.3" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="183" y="-775.3" font-family="Times,serif" font-size="9.00">Submit</text>
<text text-anchor="middle" x="183" y="-765.3" font-family="Times,serif" font-size="9.00">0.04s (0.036%)</text>
<text text-anchor="middle" x="183" y="-755.3" font-family="Times,serif" font-size="9.00">of 81.67s (73.42%)</text>
</a>
</g>
</g>
<!-- N1&#45;&gt;N14 -->
<g id="edge2" class="edge">
<title>N1-&gt;N14</title>
<g id="a_edge2"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).Submit (81.67s)">
<path fill="none" stroke="#b20f00" stroke-width="4" d="M183,-874.99C183,-857.24 183,-835.26 183,-816.77"></path>
<polygon fill="#b20f00" stroke="#b20f00" stroke-width="4" points="186.5,-816.63 183,-806.63 179.5,-816.63 186.5,-816.63"></polygon>
</a>
</g>
<g id="a_edge2-label"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).Submit (81.67s)">
<text text-anchor="middle" x="203" y="-838.3" font-family="Times,serif" font-size="14.00"> 81.67s</text>
</a>
</g>
</g>
<!-- N15 -->
<g id="node15" class="node">
<title>N15</title>
<g id="a_node15"><a xlink:title="github.com/ii64/gouring.(*IoUring).WaitCqe (9.87s)">
<polygon fill="#ede8e2" stroke="#b28c63" points="324.5,-806.5 247.5,-806.5 247.5,-748.5 324.5,-748.5 324.5,-806.5"></polygon>
<text text-anchor="middle" x="286" y="-795.3" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="286" y="-785.3" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="286" y="-775.3" font-family="Times,serif" font-size="9.00">WaitCqe</text>
<text text-anchor="middle" x="286" y="-765.3" font-family="Times,serif" font-size="9.00">0.05s (0.045%)</text>
<text text-anchor="middle" x="286" y="-755.3" font-family="Times,serif" font-size="9.00">of 9.87s (8.87%)</text>
</a>
</g>
</g>
<!-- N1&#45;&gt;N15 -->
<g id="edge9" class="edge">
<title>N1-&gt;N15</title>
<g id="a_edge9"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).WaitCqe (9.87s)">
<path fill="none" stroke="#b28c63" d="M212.2,-874.8C217.21,-868.93 222.3,-862.83 227,-857 237.89,-843.48 249.46,-828.29 259.44,-814.9"></path>
<polygon fill="#b28c63" stroke="#b28c63" points="262.28,-816.94 265.43,-806.83 256.66,-812.77 262.28,-816.94"></polygon>
</a>
</g>
<g id="a_edge9-label"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).WaitCqe (9.87s)">
<text text-anchor="middle" x="273" y="-845.8" font-family="Times,serif" font-size="14.00"> 9.87s</text>
<text text-anchor="middle" x="273" y="-830.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N17 -->
<g id="node17" class="node">
<title>N17</title>
<g id="a_node17"><a xlink:title="github.com/ii64/gouring.(*IoUring).SeenCqe (1.17s)">
<polygon fill="#edeceb" stroke="#b2b0a9" points="425.5,-806.5 348.5,-806.5 348.5,-748.5 425.5,-748.5 425.5,-806.5"></polygon>
<text text-anchor="middle" x="387" y="-795.3" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="387" y="-785.3" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="387" y="-775.3" font-family="Times,serif" font-size="9.00">SeenCqe</text>
<text text-anchor="middle" x="387" y="-765.3" font-family="Times,serif" font-size="9.00">0.02s (0.018%)</text>
<text text-anchor="middle" x="387" y="-755.3" font-family="Times,serif" font-size="9.00">of 1.17s (1.05%)</text>
</a>
</g>
</g>
<!-- N1&#45;&gt;N17 -->
<g id="edge18" class="edge">
<title>N1-&gt;N17</title>
<g id="a_edge18"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).SeenCqe (1.17s)">
<path fill="none" stroke="#b2b0a9" d="M248.07,-883.67C265.2,-876.13 283.27,-867.11 299,-857 318.09,-844.73 337.15,-828.36 352.63,-813.75"></path>
<polygon fill="#b2b0a9" stroke="#b2b0a9" points="355.13,-816.2 359.92,-806.75 350.28,-811.15 355.13,-816.2"></polygon>
</a>
</g>
<g id="a_edge18-label"><a xlink:title="main.main -> github.com/ii64/gouring.(*IoUring).SeenCqe (1.17s)">
<text text-anchor="middle" x="355" y="-838.3" font-family="Times,serif" font-size="14.00"> 1.17s</text>
</a>
</g>
</g>
<!-- N20 -->
<g id="node20" class="node">
<title>N20</title>
<g id="a_node20"><a xlink:title="github.com/ii64/gouring.PrepNop (4.61s)">
<polygon fill="#edebe8" stroke="#b2a48d" points="530.5,-795.5 453.5,-795.5 453.5,-759.5 530.5,-759.5 530.5,-795.5"></polygon>
<text text-anchor="middle" x="492" y="-784.6" font-family="Times,serif" font-size="8.00">gouring</text>
<text text-anchor="middle" x="492" y="-775.6" font-family="Times,serif" font-size="8.00">PrepNop</text>
<text text-anchor="middle" x="492" y="-766.6" font-family="Times,serif" font-size="8.00">0 of 4.61s (4.14%)</text>
</a>
</g>
</g>
<!-- N1&#45;&gt;N20 -->
<g id="edge13" class="edge">
<title>N1-&gt;N20</title>
<g id="a_edge13"><a xlink:title="main.main -> github.com/ii64/gouring.PrepNop (4.61s)">
<path fill="none" stroke="#b2a48d" d="M248.32,-896.46C286.61,-888.31 335.3,-875.5 376,-857 408,-842.46 440.84,-819.31 463.37,-801.91"></path>
<polygon fill="#b2a48d" stroke="#b2a48d" points="465.77,-804.48 471.48,-795.56 461.45,-798.97 465.77,-804.48"></polygon>
</a>
</g>
<g id="a_edge13-label"><a xlink:title="main.main -> github.com/ii64/gouring.PrepNop (4.61s)">
<text text-anchor="middle" x="449" y="-845.8" font-family="Times,serif" font-size="14.00"> 4.61s</text>
<text text-anchor="middle" x="449" y="-830.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N2 -->
<g id="node2" class="node">
<title>N2</title>
<g id="a_node2"><a xlink:title="syscall.Syscall6 (79.40s)">
<polygon fill="#edd7d5" stroke="#b21100" points="284.5,-112 81.5,-112 81.5,0 284.5,0 284.5,-112"></polygon>
<text text-anchor="middle" x="183" y="-88.8" font-family="Times,serif" font-size="24.00">syscall</text>
<text text-anchor="middle" x="183" y="-62.8" font-family="Times,serif" font-size="24.00">Syscall6</text>
<text text-anchor="middle" x="183" y="-36.8" font-family="Times,serif" font-size="24.00">78.84s (70.88%)</text>
<text text-anchor="middle" x="183" y="-10.8" font-family="Times,serif" font-size="24.00">of 79.40s (71.38%)</text>
</a>
</g>
</g>
<!-- N3 -->
<g id="node3" class="node">
<title>N3</title>
<g id="a_node3"><a xlink:title="runtime.main (111.14s)">
<polygon fill="#edd5d5" stroke="#b20000" points="227,-1030 139,-1030 139,-994 227,-994 227,-1030"></polygon>
<text text-anchor="middle" x="183" y="-1019.1" font-family="Times,serif" font-size="8.00">runtime</text>
<text text-anchor="middle" x="183" y="-1010.1" font-family="Times,serif" font-size="8.00">main</text>
<text text-anchor="middle" x="183" y="-1001.1" font-family="Times,serif" font-size="8.00">0 of 111.14s (99.92%)</text>
</a>
</g>
</g>
<!-- N3&#45;&gt;N1 -->
<g id="edge1" class="edge">
<title>N3-&gt;N1</title>
<g id="a_edge1"><a xlink:title="runtime.main -> main.main (111.14s)">
<path fill="none" stroke="#b20000" stroke-width="5" d="M183,-993.87C183,-982.73 183,-967.66 183,-953.38"></path>
<polygon fill="#b20000" stroke="#b20000" stroke-width="5" points="187.38,-953.18 183,-943.18 178.63,-953.18 187.38,-953.18"></polygon>
</a>
</g>
<g id="a_edge1-label"><a xlink:title="runtime.main -> main.main (111.14s)">
<text text-anchor="middle" x="206.5" y="-964.8" font-family="Times,serif" font-size="14.00"> 111.14s</text>
</a>
</g>
</g>
<!-- N4 -->
<g id="node4" class="node">
<title>N4</title>
<g id="a_node4"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_peek_cqe (7.38s)">
<polygon fill="#ede9e5" stroke="#b29877" points="396,-561 262,-561 262,-483 396,-483 396,-561"></polygon>
<text text-anchor="middle" x="329" y="-546.6" font-family="Times,serif" font-size="13.00">gouring</text>
<text text-anchor="middle" x="329" y="-532.6" font-family="Times,serif" font-size="13.00">(*IoUring)</text>
<text text-anchor="middle" x="329" y="-518.6" font-family="Times,serif" font-size="13.00">__io_uring_peek_cqe</text>
<text text-anchor="middle" x="329" y="-504.6" font-family="Times,serif" font-size="13.00">7.09s (6.37%)</text>
<text text-anchor="middle" x="329" y="-490.6" font-family="Times,serif" font-size="13.00">of 7.38s (6.63%)</text>
</a>
</g>
</g>
<!-- N5 -->
<g id="node5" class="node">
<title>N5</title>
<g id="a_node5"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait (81.62s)">
<polygon fill="#edd7d5" stroke="#b20f00" points="243.5,-551 122.5,-551 122.5,-493 243.5,-493 243.5,-551"></polygon>
<text text-anchor="middle" x="183" y="-539.8" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="183" y="-529.8" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="183" y="-519.8" font-family="Times,serif" font-size="9.00">__io_uring_submit_and_wait</text>
<text text-anchor="middle" x="183" y="-509.8" font-family="Times,serif" font-size="9.00">0.03s (0.027%)</text>
<text text-anchor="middle" x="183" y="-499.8" font-family="Times,serif" font-size="9.00">of 81.62s (73.38%)</text>
</a>
</g>
</g>
<!-- N9 -->
<g id="node9" class="node">
<title>N9</title>
<g id="a_node9"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_flush_sq (2.09s)">
<polygon fill="#edecea" stroke="#b2ada1" points="121.5,-432 16.5,-432 16.5,-364 121.5,-364 121.5,-432"></polygon>
<text text-anchor="middle" x="69" y="-419.2" font-family="Times,serif" font-size="11.00">gouring</text>
<text text-anchor="middle" x="69" y="-407.2" font-family="Times,serif" font-size="11.00">(*IoUring)</text>
<text text-anchor="middle" x="69" y="-395.2" font-family="Times,serif" font-size="11.00">__io_uring_flush_sq</text>
<text text-anchor="middle" x="69" y="-383.2" font-family="Times,serif" font-size="11.00">2.02s (1.82%)</text>
<text text-anchor="middle" x="69" y="-371.2" font-family="Times,serif" font-size="11.00">of 2.09s (1.88%)</text>
</a>
</g>
</g>
<!-- N5&#45;&gt;N9 -->
<g id="edge17" class="edge">
<title>N5-&gt;N9</title>
<g id="a_edge17"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait -> github.com/ii64/gouring.(*IoUring).__io_uring_flush_sq (2.09s)">
<path fill="none" stroke="#b2ada1" d="M156.56,-492.7C141.96,-477.08 123.56,-457.39 107.4,-440.09"></path>
<polygon fill="#b2ada1" stroke="#b2ada1" points="109.54,-437.25 100.15,-432.34 104.42,-442.03 109.54,-437.25"></polygon>
</a>
</g>
<g id="a_edge17-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait -> github.com/ii64/gouring.(*IoUring).__io_uring_flush_sq (2.09s)">
<text text-anchor="middle" x="147" y="-453.8" font-family="Times,serif" font-size="14.00"> 2.09s</text>
</a>
</g>
</g>
<!-- N16 -->
<g id="node16" class="node">
<title>N16</title>
<g id="a_node16"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit (79.50s)">
<polygon fill="#edd7d5" stroke="#b21100" points="226,-427 140,-427 140,-369 226,-369 226,-427"></polygon>
<text text-anchor="middle" x="183" y="-415.8" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="183" y="-405.8" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="183" y="-395.8" font-family="Times,serif" font-size="9.00">__io_uring_submit</text>
<text text-anchor="middle" x="183" y="-385.8" font-family="Times,serif" font-size="9.00">0.02s (0.018%)</text>
<text text-anchor="middle" x="183" y="-375.8" font-family="Times,serif" font-size="9.00">of 79.50s (71.47%)</text>
</a>
</g>
</g>
<!-- N5&#45;&gt;N16 -->
<g id="edge5" class="edge">
<title>N5-&gt;N16</title>
<g id="a_edge5"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait -> github.com/ii64/gouring.(*IoUring).__io_uring_submit (79.50s)">
<path fill="none" stroke="#b21100" stroke-width="4" d="M183,-492.7C183,-476.23 183,-455.23 183,-437.29"></path>
<polygon fill="#b21100" stroke="#b21100" stroke-width="4" points="186.5,-437.14 183,-427.14 179.5,-437.14 186.5,-437.14"></polygon>
</a>
</g>
<g id="a_edge5-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait -> github.com/ii64/gouring.(*IoUring).__io_uring_submit (79.50s)">
<text text-anchor="middle" x="203" y="-453.8" font-family="Times,serif" font-size="14.00"> 79.50s</text>
</a>
</g>
</g>
<!-- N6 -->
<g id="node6" class="node">
<title>N6</title>
<g id="a_node6"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_wait_cqe (9.82s)">
<polygon fill="#ede8e2" stroke="#b28c63" points="341,-680 243,-680 243,-612 341,-612 341,-680"></polygon>
<text text-anchor="middle" x="292" y="-667.2" font-family="Times,serif" font-size="11.00">gouring</text>
<text text-anchor="middle" x="292" y="-655.2" font-family="Times,serif" font-size="11.00">(*IoUring)</text>
<text text-anchor="middle" x="292" y="-643.2" font-family="Times,serif" font-size="11.00">io_uring_wait_cqe</text>
<text text-anchor="middle" x="292" y="-631.2" font-family="Times,serif" font-size="11.00">2.44s (2.19%)</text>
<text text-anchor="middle" x="292" y="-619.2" font-family="Times,serif" font-size="11.00">of 9.82s (8.83%)</text>
</a>
</g>
</g>
<!-- N6&#45;&gt;N4 -->
<g id="edge11" class="edge">
<title>N6-&gt;N4</title>
<g id="a_edge11"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_wait_cqe -> github.com/ii64/gouring.(*IoUring).__io_uring_peek_cqe (7.38s)">
<path fill="none" stroke="#b29877" d="M302.01,-611.99C305.84,-599.38 310.28,-584.74 314.44,-571.01"></path>
<polygon fill="#b29877" stroke="#b29877" points="317.85,-571.81 317.41,-561.23 311.16,-569.78 317.85,-571.81"></polygon>
</a>
</g>
<g id="a_edge11-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_wait_cqe -> github.com/ii64/gouring.(*IoUring).__io_uring_peek_cqe (7.38s)">
<text text-anchor="middle" x="329" y="-582.8" font-family="Times,serif" font-size="14.00"> 7.38s</text>
</a>
</g>
</g>
<!-- N7 -->
<g id="node7" class="node">
<title>N7</title>
<g id="a_node7"><a xlink:title="github.com/ii64/gouring.PrepRW (4.61s)">
<polygon fill="#edebe8" stroke="#b2a48d" points="543.5,-669.5 460.5,-669.5 460.5,-622.5 543.5,-622.5 543.5,-669.5"></polygon>
<text text-anchor="middle" x="502" y="-655.9" font-family="Times,serif" font-size="12.00">gouring</text>
<text text-anchor="middle" x="502" y="-642.9" font-family="Times,serif" font-size="12.00">PrepRW</text>
<text text-anchor="middle" x="502" y="-629.9" font-family="Times,serif" font-size="12.00">4.61s (4.14%)</text>
</a>
</g>
</g>
<!-- N8 -->
<g id="node8" class="node">
<title>N8</title>
<g id="a_node8"><a xlink:title="github.com/ii64/gouring.(*IoUring)._io_uring_get_sqe (3.69s)">
<polygon fill="#edebe9" stroke="#b2a794" points="104,-558.5 0,-558.5 0,-485.5 104,-485.5 104,-558.5"></polygon>
<text text-anchor="middle" x="52" y="-544.9" font-family="Times,serif" font-size="12.00">gouring</text>
<text text-anchor="middle" x="52" y="-531.9" font-family="Times,serif" font-size="12.00">(*IoUring)</text>
<text text-anchor="middle" x="52" y="-518.9" font-family="Times,serif" font-size="12.00">_io_uring_get_sqe</text>
<text text-anchor="middle" x="52" y="-505.9" font-family="Times,serif" font-size="12.00">3.22s (2.89%)</text>
<text text-anchor="middle" x="52" y="-492.9" font-family="Times,serif" font-size="12.00">of 3.69s (3.32%)</text>
</a>
</g>
</g>
<!-- N18 -->
<g id="node18" class="node">
<title>N18</title>
<g id="a_node18"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_get_sqe (3.69s)">
<polygon fill="#edebe9" stroke="#b2a794" points="97.5,-668 20.5,-668 20.5,-624 97.5,-624 97.5,-668"></polygon>
<text text-anchor="middle" x="59" y="-657.6" font-family="Times,serif" font-size="8.00">gouring</text>
<text text-anchor="middle" x="59" y="-648.6" font-family="Times,serif" font-size="8.00">(*IoUring)</text>
<text text-anchor="middle" x="59" y="-639.6" font-family="Times,serif" font-size="8.00">io_uring_get_sqe</text>
<text text-anchor="middle" x="59" y="-630.6" font-family="Times,serif" font-size="8.00">0 of 3.69s (3.32%)</text>
</a>
</g>
</g>
<!-- N10&#45;&gt;N18 -->
<g id="edge15" class="edge">
<title>N10-&gt;N18</title>
<g id="a_edge15"><a xlink:title="github.com/ii64/gouring.(*IoUring).GetSqe -> github.com/ii64/gouring.(*IoUring).io_uring_get_sqe (3.69s)">
<path fill="none" stroke="#b2a794" d="M72.73,-745.81C69.9,-725.43 66.19,-698.75 63.34,-678.26"></path>
<polygon fill="#b2a794" stroke="#b2a794" points="66.79,-677.59 61.94,-668.17 59.85,-678.55 66.79,-677.59"></polygon>
</a>
</g>
<g id="a_edge15-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).GetSqe -> github.com/ii64/gouring.(*IoUring).io_uring_get_sqe (3.69s)">
<text text-anchor="middle" x="93" y="-716.8" font-family="Times,serif" font-size="14.00"> 3.69s</text>
<text text-anchor="middle" x="93" y="-701.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N11 -->
<g id="node11" class="node">
<title>N11</title>
<g id="a_node11"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_cq_advance (1s)">
<polygon fill="#edecec" stroke="#b2b0aa" points="520,-548 414,-548 414,-496 520,-496 520,-548"></polygon>
<text text-anchor="middle" x="467" y="-536" font-family="Times,serif" font-size="10.00">gouring</text>
<text text-anchor="middle" x="467" y="-525" font-family="Times,serif" font-size="10.00">(*IoUring)</text>
<text text-anchor="middle" x="467" y="-514" font-family="Times,serif" font-size="10.00">io_uring_cq_advance</text>
<text text-anchor="middle" x="467" y="-503" font-family="Times,serif" font-size="10.00">1s (0.9%)</text>
</a>
</g>
</g>
<!-- N12 -->
<g id="node12" class="node">
<title>N12</title>
<g id="a_node12"><a xlink:title="github.com/ii64/gouring.io_uring_enter2 (79.48s)">
<polygon fill="#edd7d5" stroke="#b21100" points="226,-211 140,-211 140,-163 226,-163 226,-211"></polygon>
<text text-anchor="middle" x="183" y="-199.8" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="183" y="-189.8" font-family="Times,serif" font-size="9.00">io_uring_enter2</text>
<text text-anchor="middle" x="183" y="-179.8" font-family="Times,serif" font-size="9.00">0.08s (0.072%)</text>
<text text-anchor="middle" x="183" y="-169.8" font-family="Times,serif" font-size="9.00">of 79.48s (71.46%)</text>
</a>
</g>
</g>
<!-- N12&#45;&gt;N2 -->
<g id="edge8" class="edge">
<title>N12-&gt;N2</title>
<g id="a_edge8"><a xlink:title="github.com/ii64/gouring.io_uring_enter2 -> syscall.Syscall6 (79.40s)">
<path fill="none" stroke="#b21100" stroke-width="4" d="M183,-162.94C183,-151.4 183,-136.79 183,-122.13"></path>
<polygon fill="#b21100" stroke="#b21100" stroke-width="4" points="186.5,-122.02 183,-112.02 179.5,-122.02 186.5,-122.02"></polygon>
</a>
</g>
<g id="a_edge8-label"><a xlink:title="github.com/ii64/gouring.io_uring_enter2 -> syscall.Syscall6 (79.40s)">
<text text-anchor="middle" x="203" y="-133.8" font-family="Times,serif" font-size="14.00"> 79.40s</text>
</a>
</g>
</g>
<!-- N13 -->
<g id="node13" class="node">
<title>N13</title>
<g id="a_node13"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_cqe_seen (1.15s)">
<polygon fill="#edeceb" stroke="#b2b0a9" points="442.5,-675 359.5,-675 359.5,-617 442.5,-617 442.5,-675"></polygon>
<text text-anchor="middle" x="401" y="-663.8" font-family="Times,serif" font-size="9.00">gouring</text>
<text text-anchor="middle" x="401" y="-653.8" font-family="Times,serif" font-size="9.00">(*IoUring)</text>
<text text-anchor="middle" x="401" y="-643.8" font-family="Times,serif" font-size="9.00">io_uring_cqe_seen</text>
<text text-anchor="middle" x="401" y="-633.8" font-family="Times,serif" font-size="9.00">0.15s (0.13%)</text>
<text text-anchor="middle" x="401" y="-623.8" font-family="Times,serif" font-size="9.00">of 1.15s (1.03%)</text>
</a>
</g>
</g>
<!-- N13&#45;&gt;N11 -->
<g id="edge20" class="edge">
<title>N13-&gt;N11</title>
<g id="a_edge20"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_cqe_seen -> github.com/ii64/gouring.(*IoUring).io_uring_cq_advance (1s)">
<path fill="none" stroke="#b2b0aa" d="M416.31,-616.7C425.93,-598.92 438.4,-575.87 448.56,-557.08"></path>
<polygon fill="#b2b0aa" stroke="#b2b0aa" points="451.66,-558.7 453.34,-548.24 445.51,-555.37 451.66,-558.7"></polygon>
</a>
</g>
<g id="a_edge20-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_cqe_seen -> github.com/ii64/gouring.(*IoUring).io_uring_cq_advance (1s)">
<text text-anchor="middle" x="444" y="-582.8" font-family="Times,serif" font-size="14.00"> 1s</text>
</a>
</g>
</g>
<!-- N19 -->
<g id="node19" class="node">
<title>N19</title>
<g id="a_node19"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uringn_submit (81.62s)">
<polygon fill="#edd7d5" stroke="#b20f00" points="225,-668 141,-668 141,-624 225,-624 225,-668"></polygon>
<text text-anchor="middle" x="183" y="-657.6" font-family="Times,serif" font-size="8.00">gouring</text>
<text text-anchor="middle" x="183" y="-648.6" font-family="Times,serif" font-size="8.00">(*IoUring)</text>
<text text-anchor="middle" x="183" y="-639.6" font-family="Times,serif" font-size="8.00">io_uringn_submit</text>
<text text-anchor="middle" x="183" y="-630.6" font-family="Times,serif" font-size="8.00">0 of 81.62s (73.38%)</text>
</a>
</g>
</g>
<!-- N14&#45;&gt;N19 -->
<g id="edge3" class="edge">
<title>N14-&gt;N19</title>
<g id="a_edge3"><a xlink:title="github.com/ii64/gouring.(*IoUring).Submit -> github.com/ii64/gouring.(*IoUring).io_uringn_submit (81.62s)">
<path fill="none" stroke="#b20f00" stroke-width="4" d="M183,-748.4C183,-727.78 183,-699.77 183,-678.41"></path>
<polygon fill="#b20f00" stroke="#b20f00" stroke-width="4" points="186.5,-678.23 183,-668.23 179.5,-678.23 186.5,-678.23"></polygon>
</a>
</g>
<g id="a_edge3-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).Submit -> github.com/ii64/gouring.(*IoUring).io_uringn_submit (81.62s)">
<text text-anchor="middle" x="205" y="-716.8" font-family="Times,serif" font-size="14.00"> 81.62s</text>
<text text-anchor="middle" x="205" y="-701.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N15&#45;&gt;N6 -->
<g id="edge10" class="edge">
<title>N15-&gt;N6</title>
<g id="a_edge10"><a xlink:title="github.com/ii64/gouring.(*IoUring).WaitCqe -> github.com/ii64/gouring.(*IoUring).io_uring_wait_cqe (9.82s)">
<path fill="none" stroke="#b28c63" d="M287.3,-748.4C288.09,-731.38 289.11,-709.31 290,-690.11"></path>
<polygon fill="#b28c63" stroke="#b28c63" points="293.5,-690.24 290.47,-680.08 286.51,-689.91 293.5,-690.24"></polygon>
</a>
</g>
<g id="a_edge10-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).WaitCqe -> github.com/ii64/gouring.(*IoUring).io_uring_wait_cqe (9.82s)">
<text text-anchor="middle" x="307" y="-709.3" font-family="Times,serif" font-size="14.00"> 9.82s</text>
</a>
</g>
</g>
<!-- N21 -->
<g id="node21" class="node">
<title>N21</title>
<g id="a_node21"><a xlink:title="github.com/ii64/gouring.io_uring_enter (79.48s)">
<polygon fill="#edd7d5" stroke="#b21100" points="225,-298 141,-298 141,-262 225,-262 225,-298"></polygon>
<text text-anchor="middle" x="183" y="-287.1" font-family="Times,serif" font-size="8.00">gouring</text>
<text text-anchor="middle" x="183" y="-278.1" font-family="Times,serif" font-size="8.00">io_uring_enter</text>
<text text-anchor="middle" x="183" y="-269.1" font-family="Times,serif" font-size="8.00">0 of 79.48s (71.46%)</text>
</a>
</g>
</g>
<!-- N16&#45;&gt;N21 -->
<g id="edge6" class="edge">
<title>N16-&gt;N21</title>
<g id="a_edge6"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit -> github.com/ii64/gouring.io_uring_enter (79.48s)">
<path fill="none" stroke="#b21100" stroke-width="4" d="M183,-368.92C183,-350.42 183,-326.34 183,-308.05"></path>
<polygon fill="#b21100" stroke="#b21100" stroke-width="4" points="186.5,-308.04 183,-298.04 179.5,-308.04 186.5,-308.04"></polygon>
</a>
</g>
<g id="a_edge6-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).__io_uring_submit -> github.com/ii64/gouring.io_uring_enter (79.48s)">
<text text-anchor="middle" x="205" y="-334.8" font-family="Times,serif" font-size="14.00"> 79.48s</text>
<text text-anchor="middle" x="205" y="-319.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N17&#45;&gt;N13 -->
<g id="edge19" class="edge">
<title>N17-&gt;N13</title>
<g id="a_edge19"><a xlink:title="github.com/ii64/gouring.(*IoUring).SeenCqe -> github.com/ii64/gouring.(*IoUring).io_uring_cqe_seen (1.15s)">
<path fill="none" stroke="#b2b0a9" d="M390.04,-748.4C392.04,-729.9 394.68,-705.46 396.87,-685.2"></path>
<polygon fill="#b2b0a9" stroke="#b2b0a9" points="400.37,-685.36 397.97,-675.04 393.41,-684.61 400.37,-685.36"></polygon>
</a>
</g>
<g id="a_edge19-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).SeenCqe -> github.com/ii64/gouring.(*IoUring).io_uring_cqe_seen (1.15s)">
<text text-anchor="middle" x="418" y="-716.8" font-family="Times,serif" font-size="14.00"> 1.15s</text>
<text text-anchor="middle" x="418" y="-701.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N18&#45;&gt;N8 -->
<g id="edge16" class="edge">
<title>N18-&gt;N8</title>
<g id="a_edge16"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_get_sqe -> github.com/ii64/gouring.(*IoUring)._io_uring_get_sqe (3.69s)">
<path fill="none" stroke="#b2a794" d="M57.78,-623.75C56.91,-608.66 55.72,-587.82 54.64,-569"></path>
<polygon fill="#b2a794" stroke="#b2a794" points="58.12,-568.59 54.05,-558.81 51.13,-568.99 58.12,-568.59"></polygon>
</a>
</g>
<g id="a_edge16-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uring_get_sqe -> github.com/ii64/gouring.(*IoUring)._io_uring_get_sqe (3.69s)">
<text text-anchor="middle" x="73" y="-582.8" font-family="Times,serif" font-size="14.00"> 3.69s</text>
</a>
</g>
</g>
<!-- N19&#45;&gt;N5 -->
<g id="edge4" class="edge">
<title>N19-&gt;N5</title>
<g id="a_edge4"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uringn_submit -> github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait (81.62s)">
<path fill="none" stroke="#b20f00" stroke-width="4" d="M183,-623.75C183,-606.58 183,-581.97 183,-561.37"></path>
<polygon fill="#b20f00" stroke="#b20f00" stroke-width="4" points="186.5,-561.32 183,-551.32 179.5,-561.32 186.5,-561.32"></polygon>
</a>
</g>
<g id="a_edge4-label"><a xlink:title="github.com/ii64/gouring.(*IoUring).io_uringn_submit -> github.com/ii64/gouring.(*IoUring).__io_uring_submit_and_wait (81.62s)">
<text text-anchor="middle" x="203" y="-582.8" font-family="Times,serif" font-size="14.00"> 81.62s</text>
</a>
</g>
</g>
<!-- N20&#45;&gt;N7 -->
<g id="edge12" class="edge">
<title>N20-&gt;N7</title>
<g id="a_edge12"><a xlink:title="github.com/ii64/gouring.PrepNop -> github.com/ii64/gouring.PrepRW (4.61s)">
<path fill="none" stroke="#b2a48d" d="M493.33,-759.3C494.88,-739.2 497.49,-705.45 499.44,-680.19"></path>
<polygon fill="#b2a48d" stroke="#b2a48d" points="502.95,-680.19 500.23,-669.95 495.97,-679.65 502.95,-680.19"></polygon>
</a>
</g>
<g id="a_edge12-label"><a xlink:title="github.com/ii64/gouring.PrepNop -> github.com/ii64/gouring.PrepRW (4.61s)">
<text text-anchor="middle" x="519" y="-716.8" font-family="Times,serif" font-size="14.00"> 4.61s</text>
<text text-anchor="middle" x="519" y="-701.8" font-family="Times,serif" font-size="14.00"> (inline)</text>
</a>
</g>
</g>
<!-- N21&#45;&gt;N12 -->
<g id="edge7" class="edge">
<title>N21-&gt;N12</title>
<g id="a_edge7"><a xlink:title="github.com/ii64/gouring.io_uring_enter -> github.com/ii64/gouring.io_uring_enter2 (79.48s)">
<path fill="none" stroke="#b21100" stroke-width="4" d="M183,-261.88C183,-250.48 183,-235.11 183,-221.34"></path>
<polygon fill="#b21100" stroke="#b21100" stroke-width="4" points="186.5,-221.22 183,-211.22 179.5,-221.22 186.5,-221.22"></polygon>
</a>
</g>
<g id="a_edge7-label"><a xlink:title="github.com/ii64/gouring.io_uring_enter -> github.com/ii64/gouring.io_uring_enter2 (79.48s)">
<text text-anchor="middle" x="203" y="-232.8" font-family="Times,serif" font-size="14.00"> 79.48s</text>
</a>
</g>
</g>
</g>
</svg>
Side by side

After

(image error) Size: 32 KiB

1041
assets/sqpoll.svg Normal file
View file

File diff suppressed because it is too large Load diff
Side by side

After

(image error) Size: 62 KiB

18
bench/perf/Makefile Normal file
View file

@ -0,0 +1,18 @@
alL: run
N := 10000000
PERF_OPTS := -n $(N) -noti 5000000
GCFLAGS := $(GCFLAGS)
#GCFLAGS += -m=2
#GCFLAGS += -l=4
build:
go build -gcflags="$(GCFLAGS)" .
run: build
./perf $(PERF_OPTS) -pprofCpu pprof-nonsqpoll.cpu
./perf -sqpoll $(PERF_OPTS) -pprofCpu pprof-sqpoll.cpu
pprof:
go tool pprof -http=:9001 $(P)

127
bench/perf/main.go Normal file
View file

@ -0,0 +1,127 @@
package main
import (
"flag"
"fmt"
"os"
"runtime"
"runtime/pprof"
"syscall"
"time"
"github.com/ii64/gouring"
)
var fs = flag.NewFlagSet("perf", flag.ExitOnError)
var (
entries uint
sqPoll bool
sqThreadCpu uint
sqThreadIdle uint
N uint
noti uint
pprofCpuFilename = "pprof.cpu"
)
func init() {
fs.UintVar(&entries, "entries", 256, "Entries")
fs.BoolVar(&sqPoll, "sqpoll", false, "Enable SQPOLL")
fs.UintVar(&sqThreadCpu, "sqthreadcpu", 16, "SQ Thread CPU")
fs.UintVar(&sqThreadIdle, "sqthreadidle", 10_000, "SQ Thread idle") // milliseconds
fs.UintVar(&N, "n", 10_000, "N times")
fs.UintVar(&noti, "noti", 10_000, "Notify per attempt N")
fs.StringVar(&pprofCpuFilename, "pprofCpu", pprofCpuFilename, "pprof cpu output file")
}
func main() {
err := fs.Parse(os.Args[1:])
if err != nil {
panic(err)
}
// check entries size
if entries > uint(^uint32(0)) {
panic("entries overflow.")
}
params := &gouring.IoUringParams{}
if sqPoll {
params.Flags |= gouring.IORING_SETUP_SQPOLL
params.SqThreadCpu = uint32(sqThreadCpu)
params.SqThreadIdle = uint32(sqThreadIdle)
}
h, err := gouring.NewWithParams(uint32(entries), params)
if err != nil {
panic(err)
}
defer h.Close()
f, err := os.Create(pprofCpuFilename)
if err != nil {
panic(err)
}
fmt.Println("performing...")
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
var i, j uint
var sqe *gouring.IoUringSqe
var cqe *gouring.IoUringCqe
var submitted int
startTime := time.Now()
for i = 0; i < N; i++ {
if i%noti == 0 { // notify
fmt.Printf("n:%d e:%s\n", j, time.Now().Sub(startTime))
}
for j = 0; j < entries; j++ {
for {
// sqe could be nil if SQ is already full so we spin until we got one
sqe = h.GetSqe()
if sqe != nil {
break
}
runtime.Gosched()
}
gouring.PrepNop(sqe)
sqe.UserData.SetUint64(uint64(i + j))
}
submitted, err = h.Submit()
if err != nil {
panic(err)
}
if i%noti == 0 { // notify
fmt.Printf(" >> submitted %d\n", submitted)
}
for j = 0; j < entries; j++ {
err = h.WaitCqe(&cqe)
if err == syscall.EINTR {
continue
}
if err != nil {
panic(err)
}
if cqe == nil {
panic("cqe is nil!")
}
if cqe.Res < 0 {
panic(syscall.Errno(-cqe.Res))
}
h.SeenCqe(cqe)
}
}
_ = submitted
_ = err
}

229
const_value.go
View file

@ -1,229 +0,0 @@
package gouring
type UringSQEFlag = uint8
const (
IOSQE_FIXED_FILE_BIT = iota
IOSQE_IO_DRAIN_BIT
IOSQE_IO_LINK_BIT
IOSQE_TO_HARDLINK_BIT
IOSQE_ASYNC_BIT
IOSQE_BUFFER_SELECT_BIT
IOSQE_CQE_SKIP_BIT
)
const (
IOSQE_FIXED_FILE UringSQEFlag = 1 << IOSQE_FIXED_FILE_BIT
IOSQE_IO_DRAIN UringSQEFlag = 1 << IOSQE_IO_DRAIN_BIT
IOSQE_IO_LINK UringSQEFlag = 1 << IOSQE_IO_LINK_BIT
IOSQE_TO_HARDLINK UringSQEFlag = 1 << IOSQE_TO_HARDLINK_BIT
IOSQE_ASYNC UringSQEFlag = 1 << IOSQE_ASYNC_BIT
IOSQE_BUFFER_SELECT UringSQEFlag = 1 << IOSQE_BUFFER_SELECT_BIT
IOSQE_CQE_SKIP_SUCCESS UringSQEFlag = 1 << IOSQE_CQE_SKIP_BIT
)
//
// io_uring_setup() flags
type UringSetupFlag = uint32
const (
IORING_SETUP_IOPOLL UringSetupFlag = 1 << iota
IORING_SETUP_SQPOLL
IORING_SETUP_SQ_AFF
IORING_SETUP_SQSIZE
IORING_SETUP_CLAMP
IORING_SETUP_ATTACH_WQ
IORING_SETUP_R_DISABLED
)
//
// uring op code
type UringOpcode = uint8
const (
IORING_OP_NOP UringOpcode = iota
IORING_OP_READV
IORING_OP_WRITEV
IORING_OP_FSYNC
IORING_OP_READ_FIXED
IORING_OP_WRITE_FIXED
IORING_OP_POLL_ADD
IORING_OP_POLL_REMOVE
IORING_OP_SYNC_FILE_RANGE
IORING_OP_SENDMSG
IORING_OP_RECVMSG
IORING_OP_TIMEOUT
IORING_OP_TIMEOUT_REMOVE
IORING_OP_ACCEPT
IORING_OP_ASYNC_CANCEL
IORING_OP_LINK_TIMEOUT
IORING_OP_CONNECT
IORING_OP_FALLOCATE
IORING_OP_OPENAT
IORING_OP_CLOSE
IORING_OP_FILES_UPDATE
IORING_OP_STATX
IORING_OP_READ
IORING_OP_WRITE
IORING_OP_FADVISE
IORING_OP_MADVISE
IORING_OP_SEND
IORING_OP_RECV
IORING_OP_OPENAT2
IORING_OP_EPOLL_CTL
IORING_OP_SPLICE
IORING_OP_PROVIDE_BUFFERS
IORING_OP_REMOVE_BUFFERS
IORING_OP_TEE
IORING_OP_SHUTDOWN
IORING_OP_RENAMEAT
IORING_OP_UNLINKAT
IORING_OP_MKDIRAT
IORING_OP_SYMLINKAT
IORING_OP_LINKAT
/* this goes last, obviously */
IORING_OP_LAST
)
// sqe->fsync_flags
const IORING_FSYNC_DATASYNC uint32 = 1 << 0
// sqe->timeout_flags
const IORING_TIMEOUT_ABS uint32 = 1 << 0
// sqe->splice_flags
// extends splice(2) flags
const SPLICE_F_FD_IN_FIXED uint32 = 1 << 31
//
// cqe->flags
type UringCQEFlag = uint32
const IORING_CQE_F_BUFFER UringCQEFlag = 1 << 8
const IORING_CQE_BUFFER_SHIFT UringCQEFlag = 16
//
// Magic offsets for the application to mmap the data it needs
type UringOffset = int64
const (
IORING_OFF_SQ_RING UringOffset = 0
IORING_OFF_CQ_RING UringOffset = 0x8000000
IORING_OFF_SQES UringOffset = 0x10000000
)
//
// sq_ring->flags
type UringSQ = uint32
const (
IORING_SQ_NEED_WAKEUP UringSQ = 1 << iota // needs io_uring_enter wakeup
IORING_SQ_CQ_OVERFLOW // CQ Ring is overflow
)
//
// cq_ring->flags
type UringCQ = uint32
const IORING_CQ_EVENTFD_DISABLED = 1 << 0
//
// io_uring_enter(2) flag
type UringEnterFlag = uint32
const (
IORING_ENTER_GETEVENTS UringEnterFlag = 1 << iota
IORING_ENTER_SQ_WAKEUP
IORING_ENTER_SQ_WAIT
IORING_ENTER_EXT_ARG
)
//
// io_uring_params->features flags
type UringParamFeatureFlag = uint32
const (
IORING_FEAT_SINGLE_MMAP UringParamFeatureFlag = 1 << iota
IORING_FEAT_NODROP
IORING_FEAT_SUBMIT_STABLE
IORING_FEAT_RW_CUR_POS
IORING_FEAT_CUR_PERSONALITY
IORING_FEAT_FAST_POLL
IORING_FEAT_POLL_32BITS
IORING_FEAT_SQPOLL_NONFIXED
IORING_FEAT_EXT_ARG
IORING_FEAT_NATIVE_WORKERS
IORING_FEAT_RSRC_TAGS
IORING_FEAT_CQE_SKIP
)
//
type UringRegisterOpcode = uint
const (
IORING_REGISTER_BUFFERS UringRegisterOpcode = iota
IORING_UREGISTER_BUFFERS
IORING_REGISTER_FILES
IORING_UNREGISTER_FILES
IORING_REGISTER_EVENTFD
IORING_UNREGISTER_EVENTFD
IORING_REGISTER_FILES_UPDATE
IORING_REGISTER_EVENTFD_ASYNC
IORING_REGISTER_PROBE
IORING_REGISTER_PERSONALITY
IORING_UNREGISTER_PERSONALITY
IORING_REGISTER_RESTRICTIONS
IORING_REGISTER_ENABLE_RINGS
/* extended with tagging */
IORING_REGISTER_FILES2
IORING_REGISTER_FILES_UPDATE2
IORING_REGISTER_BUFFERS2
IORING_REGISTER_BUFFERS_UPDATE
/* set/clear io-wq thread affinities */
IORING_REGISTER_IOWQ_AFF
IORING_UNREGISTER_IOWQ_AFF
/* set/get max number of io-wq affinities */
IORING_REGISTER_IOWQ_MAX_WORKERS
// BPF soon
//
/* this goes last */
IORING_REGISTER_LAST
)
//
const IO_URING_OP_SUPPORTED = 1 << 0
//
// io_uring_restriction->opcode values
type UringRestrictionOpcode = uint32
const (
IORING_RESTRICTION_REGISTER_OP UringRestrictionOpcode = iota
IORING_RESTRICTION_SQE_OP
IORING_RESTRICTION_SQE_FLAGS_ALLOWED
IORINGN_RESTRICTION_SQE_FLAGS_REQUIRED
IORING_RESTRICTION_LAST
)

70
core.go
View file

@ -1,70 +0,0 @@
package gouring
import (
"syscall"
"github.com/pkg/errors"
)
func New(entries uint, params *IOUringParams) (*Ring, error) {
r := &Ring{}
if params != nil {
r.params = *params
}
var err error
if r.fd, err = setup(r, entries, &r.params); err != nil {
err = errors.Wrap(err, "setup")
return nil, err
}
return r, nil
}
func (r *Ring) Close() (err error) {
if err = unsetup(r); err != nil {
err = errors.Wrap(err, "close")
return
}
if err = syscall.Close(r.fd); err != nil {
err = errors.Wrap(err, "close")
return
}
// tbd..
return
}
func (r *Ring) Register(opcode UringRegisterOpcode, arg uintptr, nrArg uint) (ret int, err error) {
ret, err = register(r, opcode, arg, nrArg)
if err != nil {
err = errors.Wrap(err, "register")
return
}
return
}
func (r *Ring) Enter(toSubmit, minComplete uint, flags UringEnterFlag, sig *Sigset_t) (ret int, err error) {
ret, err = enter(r, toSubmit, minComplete, flags, sig)
if err != nil {
err = errors.Wrap(err, "enter")
return
}
return
}
//
func (r *Ring) Params() *IOUringParams {
return &r.params
}
func (r *Ring) Fd() int {
return r.fd
}
func (r *Ring) SQ() *SQRing {
return &r.sq
}
func (r *Ring) CQ() *CQRing {
return &r.cq
}

60
core_test.go
View file

@ -1,60 +0,0 @@
package gouring
import (
"strings"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCore(t *testing.T) {
ring, err := New(256, nil)
assert.NoError(t, err, "create ring")
defer func() {
err := ring.Close()
assert.NoError(t, err, "close ring")
}()
mkdata := func(i int) []byte {
return []byte("print me to stdout please" + strings.Repeat("!", i) + "\n")
}
sq := ring.SQ()
n := 5
for i := 0; i < n; i++ {
sqTail := *sq.Tail()
sqIdx := sqTail & *sq.RingMask()
sqe := sq.Get(sqIdx)
m := mkdata(i)
sqe.Opcode = IORING_OP_WRITE
sqe.Fd = int32(syscall.Stdout)
sqe.UserData = uint64(i)
sqe.Len = uint32(len(m))
sqe.SetOffset(0)
sqe.SetAddr(&m[0])
*sq.Array().Get(sqIdx) = *sq.Head() & *sq.RingMask()
*sq.Tail()++
done, err := ring.Enter(1, 1, IORING_ENTER_GETEVENTS, nil)
assert.NoError(t, err, "ring enter")
t.Logf("done %d", done)
}
// get cq
cq := ring.CQ()
for i := 0; i < int(*cq.Tail()); i++ {
cqHead := *cq.Head()
cqIdx := cqHead & *cq.RingMask()
cqe := cq.Get(cqIdx)
*cq.Head()++
t.Logf("CQE %+#v", cqe)
}
}

5
go.mod
View file

@ -2,10 +2,7 @@ module github.com/ii64/gouring
go 1.18
require (
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.7.0
)
require github.com/stretchr/testify v1.7.0
require (
github.com/davecgh/go-spew v1.1.0 // indirect

2
go.sum
View file

@ -1,7 +1,5 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

635
hdr.go Normal file
View file

@ -0,0 +1,635 @@
/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
/*
* Header file for the io_uring interface.
*
* Copyright (C) 2019 Jens Axboe
* Copyright (C) 2019 Christoph Hellwig
*/
package gouring
import "unsafe"
/*
* IO submission data structure (Submission Queue Entry)
*/
type IoUringSqe_Union1 uint64
func (u *IoUringSqe_Union1) SetOffset(v uint64) { *u = IoUringSqe_Union1(v) }
func (u *IoUringSqe_Union1) SetAddr2(v uint64) { *u = IoUringSqe_Union1(v) }
type IoUringSqe_Union2 uint64
func (u *IoUringSqe_Union2) SetAddr_Value(v uint64) { *u = IoUringSqe_Union2(v) }
func (u *IoUringSqe_Union2) SetAddr(v unsafe.Pointer) { *u = IoUringSqe_Union2((uintptr)(v)) }
func (u *IoUringSqe_Union2) SetSpliceOffsetIn(v uint64) { *u = IoUringSqe_Union2(v) }
type IoUringSqe_Union3 uint32
func (u *IoUringSqe_Union3) SetRwFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetPollEvents(v uint16) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetPoll32Events(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetSyncRangeFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetMsgFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetTimeoutFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetAcceptFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetCancelFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetOpenFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetStatxFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetFadviseAdvice(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetSpliceFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetRenameFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetUnlinkFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetHardlinkFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetXattrFlags(v uint32) { *u = IoUringSqe_Union3(v) }
func (u *IoUringSqe_Union3) SetOpFlags(v uint32) { *u = IoUringSqe_Union3(v) } //generic
func (u IoUringSqe_Union3) GetOpFlags() uint32 { return uint32(u) } //generic
type IoUringSqe_Union4 uint16
func (u *IoUringSqe_Union4) SetBufIndex(v uint16) { *u = IoUringSqe_Union4(v) }
func (u *IoUringSqe_Union4) SetBufGroup(v uint16) { *u = IoUringSqe_Union4(v) }
type IoUringSqe_Union5 uint32
func (u *IoUringSqe_Union5) SetSpliceFdIn(v int32) { *u = IoUringSqe_Union5(v) }
func (u *IoUringSqe_Union5) SetFileIndex(v uint32) { *u = IoUringSqe_Union5(v) }
type IoUringSqe struct {
Opcode IoUringOp /* type of operation for this sqe */
Flags uint8 /* IOSQE_ flags */
IoPrio uint16 /* ioprio for the request */
Fd int32 /* file descriptor to do IO on */
// union {
// __u64 off; /* offset into file */
// __u64 addr2;
// };
IoUringSqe_Union1
// union {
// __u64 addr; /* pointer to buffer or iovecs */
// __u64 splice_off_in;
// };
IoUringSqe_Union2
Len uint32 /* buffer size or number of iovecs */
// union {
// __kernel_rwf_t rw_flags;
// __u32 fsync_flags;
// __u16 poll_events; /* compatibility */
// __u32 poll32_events; /* word-reversed for BE */
// __u32 sync_range_flags;
// __u32 msg_flags;
// __u32 timeout_flags;
// __u32 accept_flags;
// __u32 cancel_flags;
// __u32 open_flags;
// __u32 statx_flags;
// __u32 fadvise_advice;
// __u32 splice_flags;
// __u32 rename_flags;
// __u32 unlink_flags;
// __u32 hardlink_flags;
// __u32 xattr_flags;
// };
IoUringSqe_Union3
UserData UserData /* data to be passed back at completion time */
/* pack this to avoid bogus arm OABI complaints */
// union {
// /* index into fixed buffers, if used */
// __u16 buf_index;
// /* for grouped buffer selection */
// __u16 buf_group;
// } __attribute__((packed));
IoUringSqe_Union4
/* personality to use, if used */
Personality uint16
// union {
// __s32 splice_fd_in;
// __u32 file_index;
// };
IoUringSqe_Union5
Addr3 uint64
__pad2 [1]uint64
}
/*
* If sqe->file_index is set to this for opcodes that instantiate a new
* direct descriptor (like openat/openat2/accept), then io_uring will allocate
* an available direct descriptor instead of having the application pass one
* in. The picked direct descriptor will be returned in cqe->res, or -ENFILE
* if the space is full.
*/
const IORING_FILE_INDEX_ALLOC = ^uint32(0)
const (
IOSQE_FIXED_FILE_BIT = iota
IOSQE_IO_DRAIN_BIT
IOSQE_IO_LINK_BIT
IOSQE_IO_HARDLINK_BIT
IOSQE_ASYNC_BIT
IOSQE_BUFFER_SELECT_BIT
IOSQE_CQE_SKIP_SUCCESS_BIT
)
/*
* sqe->flags
*/
const (
/* use fixed fileset */
IOSQE_FIXED_FILE = (1 << IOSQE_FIXED_FILE_BIT)
/* issue after inflight IO */
IOSQE_IO_DRAIN = (1 << IOSQE_IO_DRAIN_BIT)
/* links next sqe */
IOSQE_IO_LINK = (1 << IOSQE_IO_LINK_BIT)
/* like LINK, but stronger */
IOSQE_IO_HARDLINK = (1 << IOSQE_IO_HARDLINK_BIT)
/* always go async */
IOSQE_ASYNC = (1 << IOSQE_ASYNC_BIT)
/* select buffer from sqe->buf_group */
IOSQE_BUFFER_SELECT = (1 << IOSQE_BUFFER_SELECT_BIT)
/* don't post CQE if request succeeded */
IOSQE_CQE_SKIP_SUCCESS = (1 << IOSQE_CQE_SKIP_SUCCESS_BIT)
)
/*
* io_uring_setup() flags
*/
const (
IORING_SETUP_IOPOLL = (1 << 0) /* io_context is polled */
IORING_SETUP_SQPOLL = (1 << 1) /* SQ poll thread */
IORING_SETUP_SQ_AFF = (1 << 2) /* sq_thread_cpu is valid */
IORING_SETUP_CQSIZE = (1 << 3) /* app defines CQ size */
IORING_SETUP_CLAMP = (1 << 4) /* clamp SQ/CQ ring sizes */
IORING_SETUP_ATTACH_WQ = (1 << 5) /* attach to existing wq */
IORING_SETUP_R_DISABLED = (1 << 6) /* start with ring disabled */
IORING_SETUP_SUBMIT_ALL = (1 << 7) /* continue submit on error */
)
/*
* Cooperative task running. When requests complete, they often require
* forcing the submitter to transition to the kernel to complete. If this
* flag is set, work will be done when the task transitions anyway, rather
* than force an inter-processor interrupt reschedule. This avoids interrupting
* a task running in userspace, and saves an IPI.
*/
const IORING_SETUP_COOP_TASKRUN = (1 << 8)
/*
* If COOP_TASKRUN is set, get notified if task work is available for
* running and a kernel transition would be needed to run it. This sets
* IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
*/
const IORING_SETUP_TASKRUN_FLAG = (1 << 9)
const IORING_SETUP_SQE128 = (1 << 10) /* SQEs are 128 byte */
const IORING_SETUP_CQE32 = (1 << 11) /* CQEs are 32 byte */
type IoUringOp = uint8
//go:generate stringerx -type=IoUringOp
const (
IORING_OP_NOP IoUringOp = iota
IORING_OP_READV
IORING_OP_WRITEV
IORING_OP_FSYNC
IORING_OP_READ_FIXED
IORING_OP_WRITE_FIXED
IORING_OP_POLL_ADD
IORING_OP_POLL_REMOVE
IORING_OP_SYNC_FILE_RANGE
IORING_OP_SENDMSG
IORING_OP_RECVMSG
IORING_OP_TIMEOUT
IORING_OP_TIMEOUT_REMOVE
IORING_OP_ACCEPT
IORING_OP_ASYNC_CANCEL
IORING_OP_LINK_TIMEOUT
IORING_OP_CONNECT
IORING_OP_FALLOCATE
IORING_OP_OPENAT
IORING_OP_CLOSE
IORING_OP_FILES_UPDATE
IORING_OP_STATX
IORING_OP_READ
IORING_OP_WRITE
IORING_OP_FADVISE
IORING_OP_MADVISE
IORING_OP_SEND
IORING_OP_RECV
IORING_OP_OPENAT2
IORING_OP_EPOLL_CTL
IORING_OP_SPLICE
IORING_OP_PROVIDE_BUFFERS
IORING_OP_REMOVE_BUFFERS
IORING_OP_TEE
IORING_OP_SHUTDOWN
IORING_OP_RENAMEAT
IORING_OP_UNLINKAT
IORING_OP_MKDIRAT
IORING_OP_SYMLINKAT
IORING_OP_LINKAT
IORING_OP_MSG_RING
IORING_OP_FSETXATTR
IORING_OP_SETXATTR
IORING_OP_FGETXATTR
IORING_OP_GETXATTR
IORING_OP_SOCKET
IORING_OP_URING_CMD
/* this goes last, obviously */
IORING_OP_LAST
)
/*
* sqe->fsync_flags
*/
const IORING_FSYNC_DATASYNC = (1 << 0)
/*
* sqe->timeout_flags
*/
const (
IORING_TIMEOUT_ABS = (1 << 0)
IORING_TIMEOUT_UPDATE = (1 << 1)
IORING_TIMEOUT_BOOTTIME = (1 << 2)
IORING_TIMEOUT_REALTIME = (1 << 3)
IORING_LINK_TIMEOUT_UPDATE = (1 << 4)
IORING_TIMEOUT_ETIME_SUCCESS = (1 << 5)
IORING_TIMEOUT_CLOCK_MASK = (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
IORING_TIMEOUT_UPDATE_MASK = (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
)
/*
* sqe->splice_flags
* extends splice(2) flags
*/
const SPLICE_F_FD_IN_FIXED = (1 << 31) /* the last bit of __u32 */
/*
* POLL_ADD flags. Note that since sqe->poll_events is the flag space, the
* command flags for POLL_ADD are stored in sqe->len.
*
* IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if
* the poll handler will continue to report
* CQEs on behalf of the same SQE.
*
* IORING_POLL_UPDATE Update existing poll request, matching
* sqe->addr as the old user_data field.
*/
const (
IORING_POLL_ADD_MULTI = (1 << 0)
IORING_POLL_UPDATE_EVENTS = (1 << 1)
IORING_POLL_UPDATE_USER_DATA = (1 << 2)
)
/*
* ASYNC_CANCEL flags.
*
* IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key
* IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
* request 'user_data'
* IORING_ASYNC_CANCEL_ANY Match any request
*/
const (
IORING_ASYNC_CANCEL_ALL = (1 << 0)
IORING_ASYNC_CANCEL_FD = (1 << 1)
IORING_ASYNC_CANCEL_ANY = (1 << 2)
)
/*
* send/sendmsg and recv/recvmsg flags (sqe->addr2)
*
* IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send
* or receive and arm poll if that yields an
* -EAGAIN result, arm poll upfront and skip
* the initial transfer attempt.
*/
const IORING_RECVSEND_POLL_FIRST = (1 << 0)
/*
* accept flags stored in sqe->ioprio
*/
const IORING_ACCEPT_MULTISHOT = (1 << 0)
/*
* IO completion data structure (Completion Queue Entry)
*/
type IoUringCqe struct {
UserData UserData /* sqe->data submission passed back */
Res int32 /* result code for this event */
Flags uint32
/*
* If the ring is initialized with IORING_SETUP_CQE32, then this field
* contains 16-bytes of padding, doubling the size of the CQE.
*/
// __u64 big_cqe[];
// 8+4+4 == 16 , correct
}
/*
* cqe->flags
*
* IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
* IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries
* IORING_CQE_F_SOCK_NONEMPTY If set, more data to read after socket recv
*/
const (
IORING_CQE_F_BUFFER = (1 << 0)
IORING_CQE_F_MORE = (1 << 1)
IORING_CQE_F_SOCK_NONEMPTY = (1 << 2)
)
const (
IORING_CQE_BUFFER_SHIFT = 16
)
/*
* Magic offsets for the application to mmap the data it needs
*/
const (
IORING_OFF_SQ_RING = 0
IORING_OFF_CQ_RING = 0x8000000
IORING_OFF_SQES = 0x10000000
)
/*
* Filled with the offset for mmap(2)
*/
type IoSqringOffsets struct {
Head uint32
Tail uint32
RingMask uint32
RingEntries uint32
Flags uint32
Dropped uint32
Array uint32
resv1 uint32
resv2 uint64
}
/*
* sq_ring->flags
*/
const (
IORING_SQ_NEED_WAKEUP = (1 << 0) /* needs io_uring_enter wakeup */
IORING_SQ_CQ_OVERFLOW = (1 << 1) /* CQ ring is overflown */
IORING_SQ_TASKRUN = (1 << 2) /* task should enter the kernel */
)
type IoCqringOffsets struct {
Head uint32
Tail uint32
RingMask uint32
RingEntries uint32
Overflow uint32
Cqes uint32
Flags uint32
resv1 uint32
resv2 uint64
}
/*
* cq_ring->flags
*/
/* disable eventfd notifications */
const IORING_CQ_EVENTFD_DISABLED = (1 << 0)
/*
* io_uring_enter(2) flags
*/
const (
IORING_ENTER_GETEVENTS = (1 << 0)
IORING_ENTER_SQ_WAKEUP = (1 << 1)
IORING_ENTER_SQ_WAIT = (1 << 2)
IORING_ENTER_EXT_ARG = (1 << 3)
IORING_ENTER_REGISTERED_RING = (1 << 4)
)
/*
* Passed in for io_uring_setup(2). Copied back with updated info on success
*/
type IoUringParams struct {
SqEntries uint32
CqEntries uint32
Flags uint32
SqThreadCpu uint32
SqThreadIdle uint32
Features uint32
WqFd uint32
resv [3]uint32
SqOff IoSqringOffsets
CqOff IoCqringOffsets
}
/*
* io_uring_params->features flags
*/
const (
IORING_FEAT_SINGLE_MMAP = (1 << 0)
IORING_FEAT_NODROP = (1 << 1)
IORING_FEAT_SUBMIT_STABLE = (1 << 2)
IORING_FEAT_RW_CUR_POS = (1 << 3)
IORING_FEAT_CUR_PERSONALITY = (1 << 4)
IORING_FEAT_FAST_POLL = (1 << 5)
IORING_FEAT_POLL_32BITS = (1 << 6)
IORING_FEAT_SQPOLL_NONFIXED = (1 << 7)
IORING_FEAT_EXT_ARG = (1 << 8)
IORING_FEAT_NATIVE_WORKERS = (1 << 9)
IORING_FEAT_RSRC_TAGS = (1 << 10)
IORING_FEAT_CQE_SKIP = (1 << 11)
IORING_FEAT_LINKED_FILE = (1 << 12)
)
/*
* io_uring_register(2) opcodes and arguments
*/
const (
IORING_REGISTER_BUFFERS = 0
IORING_UNREGISTER_BUFFERS = 1
IORING_REGISTER_FILES = 2
IORING_UNREGISTER_FILES = 3
IORING_REGISTER_EVENTFD = 4
IORING_UNREGISTER_EVENTFD = 5
IORING_REGISTER_FILES_UPDATE = 6
IORING_REGISTER_EVENTFD_ASYNC = 7
IORING_REGISTER_PROBE = 8
IORING_REGISTER_PERSONALITY = 9
IORING_UNREGISTER_PERSONALITY = 10
IORING_REGISTER_RESTRICTIONS = 11
IORING_REGISTER_ENABLE_RINGS = 12
/* extended with tagging */
IORING_REGISTER_FILES2 = 13
IORING_REGISTER_FILES_UPDATE2 = 14
IORING_REGISTER_BUFFERS2 = 15
IORING_REGISTER_BUFFERS_UPDATE = 16
/* set/clear io-wq thread affinities */
IORING_REGISTER_IOWQ_AFF = 17
IORING_UNREGISTER_IOWQ_AFF = 18
/* set/get max number of io-wq workers */
IORING_REGISTER_IOWQ_MAX_WORKERS = 19
/* register/unregister io_uring fd with the ring */
IORING_REGISTER_RING_FDS = 20
IORING_UNREGISTER_RING_FDS = 21
/* register ring based provide buffer group */
IORING_REGISTER_PBUF_RING = 22
IORING_UNREGISTER_PBUF_RING = 23
/* this goes last */
IORING_REGISTER_LAST
)
/* io-wq worker categories */
const (
IO_WQ_BOUND = iota
IO_WQ_UNBOUND
)
/* deprecated, see struct IoUringRsrcUpdate */
type IoUringFilesUpdate struct {
Offset uint32
resv uint32
Fds uint64 // __aligned_u64/* __s32 * */
}
/*
* Register a fully sparse file space, rather than pass in an array of all
* -1 file descriptors.
*/
const IORING_RSRC_REGISTER_SPARSE = (1 << 0)
type IoUringRsrcRegister struct {
Nr uint32
Flags uint32
resv2 uint64
Data uint64 // __aligned_u64
Tags uint64 // __aligned_u64
}
type IoUringRsrcUpdate struct {
Offset uint32
resv uint32
Data uint64 // __aligned_u64
}
type IoUringRsrcUpdate2 struct {
Offset uint32
resv uint32
Data uint64 // __aligned_u64
Tags uint64 // __aligned_u64
Nr uint32
resv2 uint32
}
/* Skip updating fd indexes set to this value in the fd table */
const IORING_REGISTER_FILES_SKIP = (-2)
const IO_URING_OP_SUPPORTED = (1 << 0)
type IoUringProbeOp struct {
op uint8
resv uint8
flags uint16 /* IO_URING_OP_* flags */
resv2 uint32
}
type IoUringProbe struct {
last_op uint8 /* last opcode supported */
uint8 /* length of ops[] array below */
resv uint16
resv2 [3]uint32
ops [0]IoUringProbeOp
}
type IoUringRestriction struct {
opcode uint16
// union {
// __u8 register_op; /* IORING_RESTRICTION_REGISTER_OP */
// __u8 sqe_op; /* IORING_RESTRICTION_SQE_OP */
// __u8 sqe_flags; /* IORING_RESTRICTION_SQE_FLAGS_* */
// };
Union1 uint8
resv uint8
resv2 [3]uint32
}
type IoUringBuf struct {
Addr uint64
Len uint32
Bid uint16
resv uint16
}
type IoUringBufRing struct {
// union {
/*
* To avoid spilling into more pages than we need to, the
* ring tail is overlaid with the IoUringBuf->resv field.
*/
Anon0 struct {
resv1 uint64
resv2 uint32
resv3 uint16
Tail uint16
}
// bufs [0]IoUringBuf
// };
}
/* argument for IORING_(UN)REGISTER_PBUF_RING */
type IoUringBufReg struct {
RingAddr uint64
RingEntries uint32
Bgid uint16
Pad uint16
resv [3]uint64
}
/*
* IoUringRestriction->opcode values
*/
const (
/* Allow an io_uring_register(2) opcode */
IORING_RESTRICTION_REGISTER_OP = 0
/* Allow an sqe opcode */
IORING_RESTRICTION_SQE_OP = 1
/* Allow sqe flags */
IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2
/* Require sqe flags (these flags must be set on each submission) */
IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3
IORING_RESTRICTION_LAST
)
type IoUringGeteventsArg struct {
Sigmask uint64
SigmaskSz uint32
Pad uint32
Ts uint64
}
/*
* accept flags stored in sqe->ioprio
*/
// const IORING_ACCEPT_MULTISHOT = (1 << 0)

3
hdr_int_flags.go Normal file
View file

@ -0,0 +1,3 @@
package gouring
const INT_FLAG_REG_RING uint8 = 1

74
hdr_struct.go Normal file
View file

@ -0,0 +1,74 @@
package gouring
import "unsafe"
const (
SizeofUnsigned = unsafe.Sizeof(uint32(0))
SizeofUint32 = unsafe.Sizeof(uint32(0))
SizeofIoUringSqe = unsafe.Sizeof(IoUringSqe{})
SizeofIoUringCqe = unsafe.Sizeof(IoUringCqe{})
)
type IoUring struct {
Sq IoUringSq
Cq IoUringCq
Flags uint32
RingFd int
Features uint32
EnterRingFd int
IntFlags uint8
pad [3]uint8
pad2 uint32
}
type IoUringSq struct {
head unsafe.Pointer // *uint32
tail unsafe.Pointer // *uint32
ringMask unsafe.Pointer // *uint32
ringEntries unsafe.Pointer // *uint32
flags unsafe.Pointer // *uint32
dropped unsafe.Pointer // *uint32
Array uint32Array //ptr arith
Sqes ioUringSqeArray //ptr arith
SqeHead uint32
SqeTail uint32
RingSz uint32
RingPtr unsafe.Pointer
pad [4]uint32
}
func (sq *IoUringSq) _Head() *uint32 { return (*uint32)(sq.head) }
func (sq *IoUringSq) _Tail() *uint32 { return (*uint32)(sq.tail) }
func (sq *IoUringSq) _RingMask() *uint32 { return (*uint32)(sq.ringMask) }
func (sq *IoUringSq) _RingEntries() *uint32 { return (*uint32)(sq.ringEntries) }
func (sq *IoUringSq) _Flags() *uint32 { return (*uint32)(sq.flags) }
func (sq *IoUringSq) _Dropped() *uint32 { return (*uint32)(sq.dropped) }
type IoUringCq struct {
head unsafe.Pointer // *uint32
tail unsafe.Pointer // *uint32
ringMask unsafe.Pointer // *uint32
ringEntries unsafe.Pointer // *uint32
flags unsafe.Pointer // *uint32
overflow unsafe.Pointer // *uint32
Cqes ioUringCqeArray //ptr arith
RingSz uint32
RingPtr unsafe.Pointer
pad [4]uint32
}
func (cq *IoUringCq) _Head() *uint32 { return (*uint32)(cq.head) }
func (cq *IoUringCq) _Tail() *uint32 { return (*uint32)(cq.tail) }
func (cq *IoUringCq) _RingMask() *uint32 { return (*uint32)(cq.ringMask) }
func (cq *IoUringCq) _RingEntries() *uint32 { return (*uint32)(cq.ringEntries) }
func (cq *IoUringCq) _Flags() *uint32 { return (*uint32)(cq.flags) }
func (cq *IoUringCq) _Overflow() *uint32 { return (*uint32)(cq.overflow) }

13
hdr_test.go Normal file
View file

@ -0,0 +1,13 @@
package gouring
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDataSize(t *testing.T) {
assert.Equal(t, 64, int(SizeofIoUringSqe), "sqe data size mismatch")
assert.Equal(t, 16, int(SizeofIoUringCqe), "cqe data size mismatch")
}

63
inline_cost.py Normal file
View file

@ -0,0 +1,63 @@
#!/bin/python3
import re
import subprocess
from io import BytesIO
from typing import Tuple, List
from sys import (
stdout as sys_stdout,
stderr as sys_stderr
)
# https://dave.cheney.net/2020/05/02/mid-stack-inlining-in-go
gcflags = [
"-m=2",
]
re_cost = re.compile(rb"with cost (\d+) as:")
re_cost2 = re.compile(rb"cost (\d+) exceeds")
def main():
h = subprocess.Popen(
args=["go","build","-gcflags="+" ".join(gcflags),"."],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret: Tuple[str, str] = h.communicate()
stdout, stderr = ret
lines: List[bytes] = stderr.split(b"\n")
inlined_lines: List[bytes] = [line for line in lines if b"inline" in line]
can_inline: List[Tuple[bytes, int]] = []
cannot_inline: List[Tuple[bytes, int]] = []
for line in inlined_lines:
if b"can inline" in line:
inline_cost = int(re_cost.findall(line)[0])
can_inline += [ (line, inline_cost) ]
elif b"cannot inline" in line:
cur_cost = 0
try:
cur_cost = int(re_cost2.findall(line)[0])
except: pass
cannot_inline += [ (line, cur_cost) ]
else:
sys_stderr.write(b"[UNK] ")
sys_stderr.write(line)
sys_stderr.write(b"\n")
# sort by cost
# can_inline = sorted(can_inline, key=lambda v: v[1])
# cannot_inline = sorted(cannot_inline, key=lambda v: v[1])
for item in can_inline:
print( (str(item[1]).encode() +b"\t"+ item[0]) .decode() )
print("============")
for item in cannot_inline:
print( (str(item[1]).encode() +b"\t"+ item[0]) .decode() )
if __name__ == "__main__":
main()

136
mem_util.go
View file

@ -1,136 +0,0 @@
package gouring
import (
"syscall"
"unsafe"
"github.com/pkg/errors"
)
const (
_uint32 uint32 = 0
_sz_uint32 = unsafe.Sizeof(_uint32)
)
//go:linkname mmap syscall.mmap
func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error)
//go:linkname munmap syscall.munmap
func munmap(addr uintptr, length uintptr) (err error)
func setup(r *Ring, entries uint, parmas *IOUringParams) (ringFd int, err error) {
var sq = &r.sq
var cq = &r.cq
var p = &r.params
if ringFd, err = io_uring_setup(entries, p); err != nil {
err = errors.Wrap(err, "io_uring_setup")
return
}
if ringFd < 0 {
err = syscall.EAGAIN
return
}
featSingleMap := p.Features&IORING_FEAT_SINGLE_MMAP > 0
r.sringSz = p.SQOff.Array + p.SQEntries*uint32(_sz_uint32)
r.cringSz = p.CQOff.CQEs + p.CQEntries*uint32(_sz_cqe)
if featSingleMap {
if r.cringSz > r.sringSz {
r.sringSz = r.cringSz
}
}
// allocate ring mem
var sqRingPtr uintptr
var cqRingPtr uintptr
sqRingPtr, err = mmap(0, uintptr(r.sringSz),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
ringFd, IORING_OFF_SQ_RING)
if err != nil {
err = errors.Wrap(err, "mmap sqring")
return
}
if featSingleMap {
cqRingPtr = sqRingPtr
} else {
cqRingPtr, err = mmap(0, uintptr(r.cringSz),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
ringFd, IORING_OFF_CQ_RING)
if err != nil {
err = errors.Wrap(err, "mmap cqring")
return
}
}
r.sqRingPtr = sqRingPtr
r.cqRingPtr = cqRingPtr
//
// address Go's ring with base+offset allocated
sq.head = sqRingPtr + uintptr(p.SQOff.Head)
sq.tail = sqRingPtr + uintptr(p.SQOff.Tail)
sq.ringMask = sqRingPtr + uintptr(p.SQOff.RingMask)
sq.ringEntries = sqRingPtr + uintptr(p.SQOff.RingEntries)
sq.flags = sqRingPtr + uintptr(p.SQOff.Flags)
sq.array = uint32Array(sqRingPtr + uintptr(p.SQOff.Array))
r.sqesPtr, err = mmap(0, uintptr(p.SQEntries*uint32(_sz_sqe)),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
ringFd, IORING_OFF_SQES)
if err != nil {
err = errors.Wrap(err, "mmap sqes")
return
}
sq.sqes = sqeArray(r.sqesPtr)
//
cq.head = cqRingPtr + uintptr(p.CQOff.Head)
cq.tail = cqRingPtr + uintptr(p.CQOff.Tail)
cq.ringMask = cqRingPtr + uintptr(p.CQOff.RingMask)
cq.ringEntries = cqRingPtr + uintptr(p.CQOff.RingEntries)
cq.cqes = cqeArray(cqRingPtr + uintptr(p.CQOff.CQEs))
return
}
func unsetup(r *Ring) (err error) {
if r.sqesPtr != 0 {
if err = munmap(r.sqesPtr, uintptr(r.params.SQEntries)); err != nil {
err = errors.Wrap(err, "munmap sqes")
return
}
}
featSingleMap := r.params.Features&IORING_FEAT_SINGLE_MMAP > 0
if err = munmap(r.sqRingPtr, uintptr(r.sringSz)); err != nil {
err = errors.Wrap(err, "munmap sq")
}
if !featSingleMap || r.sqRingPtr != r.cqRingPtr { // not a single map
if err = munmap(r.cqRingPtr, uintptr(r.cringSz)); err != nil {
err = errors.Wrap(err, "munmap cq")
return
}
}
return
}
func register(r *Ring, opcode UringRegisterOpcode, arg uintptr, nrArg uint) (ret int, err error) {
if ret, err = io_uring_register(r.fd, opcode, arg, nrArg); err != nil {
err = errors.Wrap(err, "io_uring_register")
return
}
return
}
func enter(r *Ring, toSubmit, minComplete uint, flags UringEnterFlag, sig *Sigset_t) (ret int, err error) {
if ret, err = io_uring_enter(r.fd, toSubmit, minComplete, uint(flags), sig); err != nil {
err = errors.Wrap(err, "io_uring_enter")
return
}
return
}

103
prep.go Normal file
View file

@ -0,0 +1,103 @@
package gouring
import (
"syscall"
"unsafe"
)
func PrepRW(op IoUringOp, sqe *IoUringSqe, fd int,
addr unsafe.Pointer, len int, offset uint64) {
sqe.Opcode = op
sqe.Flags = 0
sqe.IoPrio = 0
sqe.Fd = int32(fd)
sqe.IoUringSqe_Union1 = IoUringSqe_Union1(offset) // union1
sqe.IoUringSqe_Union2 = *(*IoUringSqe_Union2)(unsafe.Pointer(&addr)) // union2
sqe.Len = uint32(len)
sqe.IoUringSqe_Union3 = 0 // sqe.SetOpFlags(0) // union3
sqe.IoUringSqe_Union4 = 0 // sqe.SetBufIndex(0) // union4
sqe.Personality = 0
sqe.IoUringSqe_Union5 = 0 // sqe.SetFileIndex(0) // union5
sqe.Addr3 = 0
sqe.__pad2[0] = 0
}
func PrepNop(sqe *IoUringSqe) {
PrepRW(IORING_OP_NOP, sqe, -1, nil, 0, 0)
}
func PrepTimeout(sqe *IoUringSqe, ts *syscall.Timespec, count uint32, flags uint32) {
PrepRW(IORING_OP_TIMEOUT, sqe, -1, unsafe.Pointer(ts), 1, uint64(count))
sqe.SetTimeoutFlags(flags)
}
func PrepTimeoutRemove(sqe *IoUringSqe, userDaata uint64, flags uint32) {
PrepRW(IORING_OP_TIMEOUT_REMOVE, sqe, -1, nil, 0, 0)
sqe.SetAddr_Value(userDaata)
sqe.SetTimeoutFlags(flags)
}
func PrepTimeoutUpdate(sqe *IoUringSqe, ts *syscall.Timespec, userData uint64, flags uint32) {
PrepRW(IORING_OP_TIMEOUT_REMOVE, sqe, -1, nil, 0, 0)
sqe.SetAddr_Value(userData)
sqe.SetTimeoutFlags(flags | IORING_TIMEOUT_UPDATE)
}
// ** "Syscall" OP
func PrepRead(sqe *IoUringSqe, fd int, buf *byte, nb int, offset uint64) {
PrepRW(IORING_OP_READ, sqe, fd, unsafe.Pointer(buf), nb, offset)
}
func PrepReadv(sqe *IoUringSqe, fd int,
iov *syscall.Iovec, nrVecs int,
offset uint64) {
PrepRW(IORING_OP_READV, sqe, fd, unsafe.Pointer(iov), nrVecs, offset)
}
func PrepReadv2(sqe *IoUringSqe, fd int,
iov *syscall.Iovec, nrVecs int,
offset uint64, flags uint32) {
PrepReadv(sqe, fd, iov, nrVecs, offset)
sqe.SetRwFlags(flags)
}
func PrepWrite(sqe *IoUringSqe, fd int, buf *byte, nb int, offset uint64) {
PrepRW(IORING_OP_WRITE, sqe, fd, unsafe.Pointer(buf), nb, offset)
}
func PrepWritev(sqe *IoUringSqe, fd int,
iov *syscall.Iovec, nrVecs int,
offset uint64) {
PrepRW(IORING_OP_WRITEV, sqe, fd, unsafe.Pointer(iov), nrVecs, offset)
}
func PrepWritev2(sqe *IoUringSqe, fd int,
iov *syscall.Iovec, nrVecs int,
offset uint64, flags uint32) {
PrepWritev(sqe, fd, iov, nrVecs, offset)
sqe.SetRwFlags(flags)
}
func PrepAccept(sqe *IoUringSqe, fd int, rsa *syscall.RawSockaddrAny, rsaSz *uintptr, flags uint) {
// *rsaSz = syscall.SizeofSockaddrAny // leave this out to caller?
PrepRW(IORING_OP_ACCEPT, sqe, fd, unsafe.Pointer(rsa), 0, uint64(uintptr(unsafe.Pointer(rsaSz))))
sqe.SetAcceptFlags(uint32(flags))
}
func PrepClose(sqe *IoUringSqe, fd int) {
PrepRW(IORING_OP_CLOSE, sqe, fd, nil, 0, 0)
}
func PrepRecvmsg(sqe *IoUringSqe, fd int, msg *syscall.Msghdr, flags uint) {
PrepRW(IORING_OP_RECVMSG, sqe, fd, unsafe.Pointer(msg), 1, 0)
sqe.SetMsgFlags(uint32(flags))
}
func PrepSendmsg(sqe *IoUringSqe, fd int, msg *syscall.Msghdr, flags uint) {
PrepRW(IORING_OP_SENDMSG, sqe, fd, unsafe.Pointer(msg), 1, 0)
sqe.SetMsgFlags(uint32(flags))
}
// ** Multishot
func PrepMultishotAccept(sqe *IoUringSqe, fd int, rsa *syscall.RawSockaddrAny, rsaSz *uintptr, flags uint) {
PrepAccept(sqe, fd, rsa, rsaSz, flags)
sqe.IoPrio |= IORING_ACCEPT_MULTISHOT
}

478
queue.go Normal file
View file

@ -0,0 +1,478 @@
package gouring
import (
"runtime"
"sync/atomic"
"syscall"
"unsafe"
)
const LIBURING_UDATA_TIMEOUT uint64 = ^uint64(0)
/*
* Returns true if we're not using SQ thread (thus nobody submits but us)
* or if IORING_SQ_NEED_WAKEUP is set, so submit thread must be explicitly
* awakened. For the latter case, we set the thread wakeup flag.
*/
func (ring *IoUring) sq_ring_needs_enter(flags *uint32) bool {
if ring.Flags&IORING_SETUP_SQPOLL == 0 {
return true
}
// FIXME: io_uring_smp_mb
if atomic.LoadUint32(ring.Sq._Flags())&IORING_SQ_NEED_WAKEUP != 0 {
*flags |= IORING_ENTER_SQ_WAKEUP
return true
}
return false
}
func (ring *IoUring) cq_ring_needs_flush() bool {
return atomic.LoadUint32(ring.Sq._Flags())&(IORING_SQ_CQ_OVERFLOW|IORING_SQ_TASKRUN) != 0
}
func (ring *IoUring) cq_ring_needs_enter() bool {
return (ring.Flags&IORING_SETUP_IOPOLL != 0) || ring.cq_ring_needs_flush()
}
type get_data struct {
submit uint32
waitNr uint32
getFlags uint32
sz int32
arg unsafe.Pointer
}
func (ring *IoUring) _io_uring_get_cqe(cqePtr **IoUringCqe, data *get_data) (err error) {
var cqe *IoUringCqe
var looped = false
var ret int
for {
var needEnter = false
var flags uint32 = 0
var nrAvail uint32 = 0
err = ring.__io_uring_peek_cqe(&cqe, &nrAvail)
if err != nil {
break
}
if cqe != nil && data.waitNr == 0 && data.submit == 0 {
if looped || !ring.cq_ring_needs_enter() {
err = syscall.EAGAIN
break
}
needEnter = true
}
if data.waitNr > nrAvail || needEnter {
flags = IORING_ENTER_GETEVENTS | data.getFlags
needEnter = true
}
if data.submit > 0 && ring.sq_ring_needs_enter(&flags) {
needEnter = true
}
if !needEnter {
break
}
if ring.IntFlags&INT_FLAG_REG_RING != 0 {
flags |= IORING_ENTER_REGISTERED_RING
}
ret, err = io_uring_enter2(ring.EnterRingFd, data.submit, data.waitNr, flags, (*Sigset_t)(data.arg), data.sz)
if err != nil {
break
}
data.submit -= uint32(ret)
if cqe != nil {
break
}
looped = true
}
*cqePtr = cqe
return
}
func (ring *IoUring) __io_uring_get_cqe(cqePtr **IoUringCqe, submit uint32, waitNr uint32, sigmask *Sigset_t) error {
data := &get_data{
submit: submit,
waitNr: waitNr,
getFlags: 0,
sz: NSIG / 8,
arg: unsafe.Pointer(sigmask),
}
return ring._io_uring_get_cqe(cqePtr, data)
}
/*
* Fill in an array of IO completions up to count, if any are available.
* Returns the amount of IO completions filled.
*/
func (ring *IoUring) io_uring_peek_batch_cqe(cqes []*IoUringCqe, count uint32) uint32 {
var ready uint32
var overflowChecked = false
var shift = 0
if ring.Flags&IORING_SETUP_CQE32 != 0 {
shift = 1
}
again:
ready = ring.io_uring_cq_ready()
if ready > 0 {
var head = *ring.Cq._Head()
var mask = *ring.Cq._RingMask()
var last uint32
if count > ready {
count = ready
}
last = head + count
var i uintptr = 0
for head != last {
cqes[i] = ioUringCqeArray_Index(ring.Cq.Cqes, uintptr((head&mask)<<uint32(shift)))
i++
head++
}
return count
}
if overflowChecked {
goto done
}
if ring.cq_ring_needs_flush() {
var flags uint32 = IORING_ENTER_GETEVENTS
if ring.IntFlags&INT_FLAG_REG_RING != 0 {
flags |= IORING_ENTER_REGISTERED_RING
}
io_uring_enter(ring.EnterRingFd, 0, 0, flags, nil)
overflowChecked = true
goto again
}
done:
return 0
}
/*
* Sync internal state with kernel ring state on the SQ side. Returns the
* number of pending items in the SQ ring, for the shared ring.
*/
func (ring *IoUring) __io_uring_flush_sq() uint32 {
sq := &ring.Sq
var mask = *sq._RingMask()
var ktail = *sq._Tail()
var toSubmit = sq.SqeTail - sq.SqeHead
if toSubmit < 1 {
goto out
}
/*
* Fill in sqes that we have queued up, adding them to the kernel ring
*/
for ; toSubmit > 0; toSubmit-- {
*uint32Array_Index(sq.Array, uintptr(ktail&mask)) = sq.SqeHead & mask
ktail++
sq.SqeHead++
}
/*
* Ensure that the kernel sees the SQE updates before it sees the tail
* update.
*/
atomic.StoreUint32(sq._Tail(), ktail)
out:
/*
* This _may_ look problematic, as we're not supposed to be reading
* SQ->head without acquire semantics. When we're in SQPOLL mode, the
* kernel submitter could be updating this right now. For non-SQPOLL,
* task itself does it, and there's no potential race. But even for
* SQPOLL, the load is going to be potentially out-of-date the very
* instant it's done, regardless or whether or not it's done
* atomically. Worst case, we're going to be over-estimating what
* we can submit. The point is, we need to be able to deal with this
* situation regardless of any perceived atomicity.
*/
return ktail - *sq._Head()
}
/*
* If we have kernel support for IORING_ENTER_EXT_ARG, then we can use that
* more efficiently than queueing an internal timeout command.
*/
func (ring *IoUring) io_uring_wait_cqes_new(cqePtr **IoUringCqe, waitNtr uint32, ts *syscall.Timespec, sigmask *Sigset_t) error {
arg := &IoUringGeteventsArg{
Sigmask: uint64(uintptr(unsafe.Pointer(sigmask))),
SigmaskSz: NSIG / 8,
Ts: uint64(uintptr(unsafe.Pointer(ts))),
}
data := &get_data{
waitNr: waitNtr,
getFlags: IORING_ENTER_EXT_ARG,
sz: int32(unsafe.Sizeof(arg)),
}
return ring._io_uring_get_cqe(cqePtr, data)
}
/*
* Like io_uring_wait_cqe(), except it accepts a timeout value as well. Note
* that an sqe is used internally to handle the timeout. For kernel doesn't
* support IORING_FEAT_EXT_ARG, applications using this function must never
* set sqe->user_data to LIBURING_UDATA_TIMEOUT!
*
* For kernels without IORING_FEAT_EXT_ARG (5.10 and older), if 'ts' is
* specified, the application need not call io_uring_submit() before
* calling this function, as we will do that on its behalf. From this it also
* follows that this function isn't safe to use for applications that split SQ
* and CQ handling between two threads and expect that to work without
* synchronization, as this function manipulates both the SQ and CQ side.
*
* For kernels with IORING_FEAT_EXT_ARG, no implicit submission is done and
* hence this function is safe to use for applications that split SQ and CQ
* handling between two threads.
*/
func (ring *IoUring) __io_uring_submit_timeout(waitNr uint32, ts *syscall.Timespec) (ret int, err error) {
sqe := ring.io_uring_get_sqe()
if sqe == nil {
ret, err = ring.io_uring_submit()
if err != nil {
return
}
sqe = ring.io_uring_get_sqe()
if sqe == nil {
err = syscall.EAGAIN
return
}
}
PrepTimeout(sqe, ts, waitNr, 0)
sqe.UserData.SetUint64(LIBURING_UDATA_TIMEOUT)
ret = int(ring.__io_uring_flush_sq())
return
}
func (ring *IoUring) io_uring_wait_cqes(cqePtr **IoUringCqe, waitNtr uint32, ts *syscall.Timespec, sigmask *Sigset_t) (err error) {
var toSubmit = 0
if ts != nil {
if ring.Features&IORING_FEAT_EXT_ARG != 0 {
err = ring.io_uring_wait_cqes_new(cqePtr, waitNtr, ts, sigmask)
return
}
toSubmit, err = ring.__io_uring_submit_timeout(waitNtr, ts)
if err != nil {
return
}
}
err = ring.__io_uring_get_cqe(cqePtr, uint32(toSubmit), waitNtr, sigmask)
return
}
func (ring *IoUring) io_uring_submit_and_wait_timeout(cqePtr **IoUringCqe, waitNtr uint32, ts *syscall.Timespec, sigmask *Sigset_t) (err error) {
var toSubmit int
if ts != nil {
if ring.Features&IORING_FEAT_EXT_ARG != 0 {
arg := IoUringGeteventsArg{
Sigmask: uint64(uintptr(unsafe.Pointer(sigmask))),
SigmaskSz: NSIG / 8,
Ts: uint64(uintptr(unsafe.Pointer(ts))),
}
data := &get_data{
submit: ring.__io_uring_flush_sq(),
waitNr: waitNtr,
getFlags: IORING_ENTER_EXT_ARG,
sz: int32(unsafe.Sizeof(arg)),
arg: unsafe.Pointer(&arg),
}
return ring._io_uring_get_cqe(cqePtr, data)
}
toSubmit, err = ring.__io_uring_submit_timeout(waitNtr, ts)
if err != nil {
return
}
} else {
toSubmit = int(ring.__io_uring_flush_sq())
}
err = ring.__io_uring_get_cqe(cqePtr, uint32(toSubmit), waitNtr, sigmask)
return
}
/*
* See io_uring_wait_cqes() - this function is the same, it just always uses
* '1' as the wait_nr.
*/
func (ring *IoUring) io_uring_wait_cqe_timeout(cqePtr **IoUringCqe, ts *syscall.Timespec) error {
return ring.io_uring_wait_cqes(cqePtr, 1, ts, nil)
}
/*
* Submit sqes acquired from io_uring_get_sqe() to the kernel.
*
* Returns number of sqes submitted
*/
func (ring *IoUring) io_uring_submit() (int, error) {
return ring.__io_uring_submit_and_wait(0)
}
/*
* Like io_uring_submit(), but allows waiting for events as well.
*
* Returns number of sqes submitted
*/
func (ring *IoUring) io_uring_submit_and_wait(waitNtr uint32) (int, error) {
return ring.__io_uring_submit_and_wait(waitNtr)
}
func (ring *IoUring) __io_uring_submit_and_wait(waitNr uint32) (int, error) {
return ring.__io_uring_submit(ring.__io_uring_flush_sq(), waitNr)
}
func (ring *IoUring) __io_uring_submit(submitted uint32, waitNr uint32) (ret int, err error) {
var flags uint32 = 0
if ring.sq_ring_needs_enter(&flags) || waitNr != 0 {
if waitNr != 0 || ring.Flags&IORING_SETUP_IOPOLL != 0 {
flags |= IORING_ENTER_GETEVENTS
}
if ring.IntFlags&INT_FLAG_REG_RING != 0 {
flags |= IORING_ENTER_REGISTERED_RING
}
ret, err = io_uring_enter(ring.EnterRingFd, submitted, waitNr, flags, nil)
} else {
ret = int(submitted)
}
return
}
func (ring *IoUring) io_uring_get_sqe() *IoUringSqe {
return ring._io_uring_get_sqe()
}
/*
* Return an sqe to fill. Application must later call io_uring_submit()
* when it's ready to tell the kernel about it. The caller may call this
* function multiple times before calling io_uring_submit().
*
* Returns a vacant sqe, or NULL if we're full.
*/
func (ring *IoUring) _io_uring_get_sqe() (sqe *IoUringSqe) {
sq := &ring.Sq
var head = atomic.LoadUint32(sq._Head())
var next = sq.SqeTail + 1
var shift uint32 = 0
if ring.Flags&IORING_SETUP_SQE128 != 0 {
shift = 1
}
if next-head <= *sq._RingEntries() {
sqe = ioUringSqeArray_Index(sq.Sqes, uintptr((sq.SqeTail&*sq._RingMask())<<shift))
sq.SqeTail = next
return
}
sqe = nil
return
}
func (ring *IoUring) io_uring_cq_ready() uint32 {
return atomic.LoadUint32(ring.Cq._Tail()) - *ring.Cq._Head()
}
func (ring *IoUring) __io_uring_peek_cqe(cqePtr **IoUringCqe, nrAvail *uint32) error {
var cqe *IoUringCqe
var err int32 = 0
var avail int
var mask = *ring.Cq._RingMask()
var shift uint32 = 0
if ring.Flags&IORING_SETUP_CQE32 != 0 {
shift = 1
}
for {
var tail = atomic.LoadUint32(ring.Cq._Tail())
var head = *ring.Cq._Head()
cqe = nil
avail = int(tail - head)
if avail < 1 {
break
}
cqe = ioUringCqeArray_Index(ring.Cq.Cqes, uintptr((head&mask)<<shift))
if ring.Features&IORING_FEAT_EXT_ARG == 0 &&
cqe.UserData.GetUint64() == LIBURING_UDATA_TIMEOUT {
if cqe.Res < 0 {
err = cqe.Res
}
ring.io_uring_cq_advance(1)
if err == 0 {
// yields G
runtime.Gosched()
continue
}
cqe = nil
}
break
}
*cqePtr = cqe
if nrAvail != nil {
*nrAvail = uint32(avail)
}
if err == 0 {
return nil
}
return syscall.Errno(-err)
}
func (ring *IoUring) io_uring_cq_advance(nr uint32) {
if nr > 0 {
atomic.StoreUint32(ring.Cq._Head(), *ring.Cq._Head()+nr)
}
}
/*
* Return an IO completion, waiting for 'wait_nr' completions if one isn't
* readily available. Returns 0 with cqe_ptr filled in on success, -errno on
* failure.
*/
func (ring *IoUring) io_uring_wait_cqe_nr(cqePtr **IoUringCqe, waitNr uint32) error {
return ring.__io_uring_get_cqe(cqePtr, 0, waitNr, nil)
}
/*
* Return an IO completion, if one is readily available. Returns 0 with
* cqe_ptr filled in on success, -errno on failure.
*/
func (ring *IoUring) io_uring_peek_cqe(cqePtr **IoUringCqe) error {
err := ring.__io_uring_peek_cqe(cqePtr, nil)
if err == nil && *cqePtr != nil {
return nil
}
return ring.io_uring_wait_cqe_nr(cqePtr, 0)
}
/*
* Return an IO completion, waiting for it if necessary. Returns 0 with
* cqe_ptr filled in on success, -errno on failure.
*/
func (ring *IoUring) io_uring_wait_cqe(cqePtr **IoUringCqe) error {
err := ring.__io_uring_peek_cqe(cqePtr, nil)
if err == nil && *cqePtr != nil {
return nil
}
return ring.io_uring_wait_cqe_nr(cqePtr, 1)
}
/*
* Must be called after io_uring_{peek,wait}_cqe() after the cqe has
* been processed by the application.
*/
func (ring *IoUring) io_uring_cqe_seen(cqe *IoUringCqe) {
if cqe != nil {
ring.io_uring_cq_advance(1)
}
}

195
queue/queue.go
View file

@ -1,195 +0,0 @@
package queue
// Modified form
// https://github.com/iceber/io_uring-go types.go
import (
"errors"
"runtime"
"sync"
"sync/atomic"
"syscall"
"github.com/ii64/gouring"
)
var (
ErrQueueClosed = errors.New("queue closed")
)
type Queue struct {
ring *gouring.Ring
sq *gouring.SQRing
cq *gouring.CQRing
sqeHead uint32
sqeTail uint32
sMx sync.Mutex
cqMx sync.RWMutex // tbd...
clq uint32
}
func New(ring *gouring.Ring) *Queue {
if ring == nil {
return nil
}
sq := ring.SQ()
cq := ring.CQ()
return &Queue{
ring: ring,
sq: sq,
cq: cq,
clq: 0,
}
}
func (q *Queue) Close() error {
atomic.StoreUint32(&q.clq, 1)
return nil
}
func (q *Queue) precheck() (err error) {
if clq := atomic.LoadUint32(&q.clq); clq == 1 {
err = ErrQueueClosed
return
}
return
}
//
func (q *Queue) _getSQEntry() *gouring.SQEntry {
head := atomic.LoadUint32(q.sq.Head())
next := q.sqeTail + 1
if (next - head) <= atomic.LoadUint32(q.sq.RingEntries()) {
sqe := q.sq.Get(q.sqeTail & atomic.LoadUint32(q.sq.RingMask()))
q.sqeTail = next
sqe.Reset()
return sqe
}
return nil
}
func (q *Queue) GetSQEntry() (sqe *gouring.SQEntry) {
for {
sqe = q._getSQEntry()
if sqe != nil {
return
}
runtime.Gosched()
}
}
func (q *Queue) sqFallback(d uint32) {
q.sqeTail -= d
}
func (q *Queue) sqFlush() uint32 {
if q.sqeHead == q.sqeTail {
return atomic.LoadUint32(q.sq.Tail()) - atomic.LoadUint32(q.sq.Head())
}
ktail := atomic.LoadUint32(q.sq.Tail())
for toSubmit := q.sqeTail - q.sqeHead; toSubmit > 0; toSubmit-- {
*q.sq.Array().Get(ktail & (*q.sq.RingMask())) = q.sqeHead & (*q.sq.RingMask())
ktail++
q.sqeHead++
}
atomic.StoreUint32(q.sq.Tail(), ktail)
return ktail - *q.sq.Head()
}
func (q *Queue) isNeedEnter(flags *uint32) bool {
if (q.ring.Params().Features & gouring.IORING_SETUP_SQPOLL) > 0 {
return true
}
if q.sq.IsNeedWakeup() {
*flags |= gouring.IORING_SQ_NEED_WAKEUP
return true
}
return false
}
func (q *Queue) Submit() (ret int, err error) {
q.sMx.Lock()
defer q.sMx.Unlock()
submitted := q.sqFlush()
var flags uint32
if !q.isNeedEnter(&flags) || submitted == 0 {
return
}
if (q.ring.Params().Flags & gouring.IORING_SETUP_IOPOLL) > 0 {
flags |= gouring.IORING_ENTER_GETEVENTS
}
ret, err = q.ring.Enter(uint(submitted), 0, flags, nil)
return
}
//
func (q *Queue) cqPeek() (cqe *gouring.CQEntry) {
head := atomic.LoadUint32(q.cq.Head())
if head != atomic.LoadUint32(q.cq.Tail()) {
cqe = q.cq.Get(head & atomic.LoadUint32(q.cq.RingMask()))
}
return
}
func (q *Queue) cqAdvance(d uint32) {
if d != 0 {
atomic.AddUint32(q.cq.Head(), d)
}
}
func (q *Queue) GetCQEvent(wait bool) (cqe *gouring.CQEntry, err error) {
if err = q.precheck(); err != nil {
return
}
var tryPeeks int
for {
if cqe = q.cqPeek(); cqe != nil {
q.cqAdvance(1)
return
}
if !wait && !q.sq.IsCQOverflow() {
err = syscall.EAGAIN
return
}
if q.sq.IsCQOverflow() {
_, err = q.ring.Enter(0, 0, gouring.IORING_ENTER_GETEVENTS, nil)
if err != nil {
return
}
continue
}
if tryPeeks++; tryPeeks < 3 {
runtime.Gosched()
continue
}
// implement interrupt
}
}
func (q *Queue) Run(f func(cqe *gouring.CQEntry)) {
for q.precheck() == nil {
cqe, err := q.GetCQEvent(true)
if cqe == nil || err != nil {
// fmt.Printf("run error: %+#v\n", err)
if err == ErrQueueClosed {
return
}
continue
}
f(cqe)
}
}

72
queue/queue_test.go
View file

@ -1,72 +0,0 @@
package queue
import (
"fmt"
"strings"
"sync"
"syscall"
"testing"
"github.com/ii64/gouring"
"github.com/stretchr/testify/assert"
)
func write(sqe *gouring.SQEntry, fd int, b []byte) {
sqe.Opcode = gouring.IORING_OP_WRITE
sqe.Fd = int32(fd)
sqe.Len = uint32(len(b))
sqe.SetOffset(0)
// *sqe.Addr() = (uint64)(uintptr(unsafe.Pointer(&b[0])))
sqe.SetAddr(&b[0])
}
func TestQueue(t *testing.T) {
ring, err := gouring.New(256, nil)
assert.NoError(t, err, "create ring")
defer func() {
err := ring.Close()
assert.NoError(t, err, "close ring")
}()
mkdata := func(i int) []byte {
return []byte("queue pls" + strings.Repeat("!", i) + fmt.Sprintf("%d", i) + "\n")
}
N := 64 + 64
var wg sync.WaitGroup
btests := [][]byte{}
for i := 0; i < N; i++ {
btests = append(btests, mkdata(i))
}
wg.Add(N)
// create new queue
q := New(ring)
go func() {
for i, b := range btests {
sqe := q.GetSQEntry()
sqe.UserData = uint64(i)
// sqe.Flags = gouring.IOSQE_IO_DRAIN
write(sqe, syscall.Stdout, b)
if (i+1)%2 == 0 {
n, err := q.Submit()
assert.NoError(t, err, "queue submit")
assert.Equal(t, n, 2, "submit count mismatch")
fmt.Printf("submitted %d\n", n)
}
}
}()
go func() {
q.Run(func(cqe *gouring.CQEntry) {
defer wg.Done()
assert.Condition(t, func() (success bool) {
return cqe.UserData < uint64(len(btests))
}, "userdata is set with the btest index")
assert.Conditionf(t, func() (success bool) {
return len(btests[cqe.UserData]) == int(cqe.Res)
}, "OP_WRITE result mismatch: %+#v", cqe)
})
}()
wg.Wait()
}

228
queue_test.go Normal file
View file

@ -0,0 +1,228 @@
package gouring
import (
"context"
"fmt"
"os"
"reflect"
"runtime"
"sync"
"syscall"
"testing"
"unsafe"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRingQueueGetSQE(t *testing.T) {
h := testNewIoUring(t, 256, 0)
defer h.Close()
assert.NotEqual(t, 0, h.RingFd)
assert.NotEqual(t, 0, h.EnterRingFd)
sqe := h.io_uring_get_sqe()
assert.NotNil(t, sqe)
fmt.Printf("%+#v\n", sqe)
}
// func TestRingSqpollOnly(t *testing.T) {
// h := testNewIoUringWithParams(t, 256, &IoUringParams{
// Flags: IORING_SETUP_SQPOLL,
// SqThreadCpu: 10, // ms
// SqThreadIdle: 10_000,
// })
// for i := 0; i < 10; i++ {
// sqe := h.GetSqe()
// PrepNop(sqe)
// }
// h.Submit()
// var cqe *IoUringCqe
// for {
// h.WaitCqe(&cqe)
// spew.Dump(cqe)
// h.SeenCqe(cqe)
// }
// }
func TestRingQueueOrderRetrieval(t *testing.T) {
const entries = 256
h := testNewIoUring(t, entries, 0)
defer h.Close()
var i uint64
for i = 0; i < entries; i++ {
sqe := h.GetSqe()
PrepNop(sqe)
sqe.UserData.SetUint64(i)
sqe.Flags |= IOSQE_IO_LINK // ordered
}
submitted, err := h.SubmitAndWait(entries)
require.NoError(t, err)
require.Equal(t, int(entries), submitted)
var cqe *IoUringCqe
for i = 0; i < entries; i++ {
err = h.WaitCqe(&cqe)
require.NoError(t, err)
require.NotNil(t, cqe)
require.Equal(t, i, cqe.UserData.GetUint64())
h.SeenCqe(cqe)
}
}
func TestRingQueueSubmitSingleConsumer(t *testing.T) {
type opt struct {
name string
jobCount int
entries uint32
p IoUringParams
}
ts := []opt{
{"def-1-256", 1, 256, IoUringParams{}},
{"def-128-256", 256, 256, IoUringParams{}}, // passed 128
{"def-128-256", 256, 256, IoUringParams{}}, // passed 128
{"def-8-256", 8, 256, IoUringParams{}},
{"def-16-256", 16, 256, IoUringParams{}},
{"def-32-256", 32, 256, IoUringParams{}},
{"def-64-256", 64, 256, IoUringParams{}},
{"def-128-256", 128, 256, IoUringParams{}},
{"def-128+1-256", 128 + 1, 256, IoUringParams{}}, // passed 128
{"def-128+2-256", 128 + 2, 256, IoUringParams{}}, // passed 128
{"def-256-256", 256, 256, IoUringParams{}},
{"sqpoll-127-256", 127, 256, IoUringParams{Flags: IORING_SETUP_SQPOLL, SqThreadCpu: 4, SqThreadIdle: 10_000}},
{"sqpoll-128+2-256", 128 + 2, 256, IoUringParams{Flags: IORING_SETUP_SQPOLL, SqThreadCpu: 4, SqThreadIdle: 10_000}},
{"sqpoll-256-256", 256, 256, IoUringParams{Flags: IORING_SETUP_SQPOLL, SqThreadCpu: 4, SqThreadIdle: 10_000}},
// we can have other test for queue overflow.
}
for _, tc := range ts {
t.Run(tc.name, func(t *testing.T) {
ftmp, err := os.CreateTemp(os.TempDir(), "test_iouring_queue_sc_*")
require.NoError(t, err)
defer ftmp.Close()
fdTemp := ftmp.Fd()
consumer := func(h *IoUring, ctx context.Context, wg *sync.WaitGroup) {
var cqe *IoUringCqe
var err error
defer func() {
rec := recover()
if rec != nil {
spew.Dump(cqe)
}
}()
for ctx.Err() == nil {
err = h.io_uring_wait_cqe(&cqe)
if err == syscall.EINTR {
// ignore INTR
continue
}
if err != nil {
panic(err)
}
if cqe.Res < 0 {
panic(syscall.Errno(-cqe.Res))
}
// cqe data check
if int(cqe.Res) < len("data ") {
panic(fmt.Sprintf("write less that it should"))
}
if (cqe.UserData.GetUintptr()>>(8<<2))&0xff == 0x00 {
panic(fmt.Sprintf("cqe userdata should contain canonical address got %+#v", cqe.UserData))
}
bufPtr := (*[]byte)(cqe.UserData.GetUnsafe())
buf := *bufPtr // deref check
_ = buf
// fmt.Printf("%+#v %s", buf, buf)
h.io_uring_cqe_seen(cqe) // necessary
wg.Done()
}
}
submit := func(t *testing.T, opt *IoUringParams, h *IoUring, expectedSubmitCount int) {
submitted, err := h.io_uring_submit()
assert.NoError(t, err)
if opt.Flags&IORING_SETUP_SQPOLL == 0 {
assert.Equal(t, expectedSubmitCount, submitted)
}
}
t.Run("submit_single", func(t *testing.T) {
var wg sync.WaitGroup
h := testNewIoUringWithParams(t, 256, &tc.p)
defer h.Close()
wg.Add(tc.jobCount)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go consumer(h, ctx, &wg)
for i := 0; i < tc.jobCount; i++ {
var sqe *IoUringSqe
for { // sqe could be nil if SQ is already full so we spin until we got one
sqe = h.io_uring_get_sqe()
if sqe != nil {
break
}
}
var buf = new([]byte)
*buf = append(*buf, []byte(fmt.Sprintf("data %d\n", i))...)
reflect.ValueOf(buf) // escape the `buf`
PrepWrite(sqe, int(fdTemp), &(*buf)[0], len((*buf)), 0)
runtime.KeepAlive(buf)
sqe.UserData.SetUnsafe(unsafe.Pointer(buf))
// submit
submit(t, &tc.p, h, 1)
}
runtime.GC()
wg.Wait()
})
t.Run("submit_bulk", func(t *testing.T) {
var wg sync.WaitGroup
h := testNewIoUringWithParams(t, 256, &tc.p)
defer h.Close()
wg.Add(tc.jobCount)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go consumer(h, ctx, &wg)
for i := 0; i < tc.jobCount; i++ {
sqe := h.io_uring_get_sqe()
if sqe == nil {
// spin until we got one
continue
}
buf := new([]byte)
*buf = append(*buf, []byte(fmt.Sprintf("data %d\n", i))...)
PrepWrite(sqe, int(fdTemp), &(*buf)[0], len((*buf)), 0)
sqe.UserData.SetUnsafe(unsafe.Pointer(buf))
}
submit(t, &tc.p, h, tc.jobCount)
runtime.GC()
wg.Wait()
})
})
}
}

276
register.go Normal file
View file

@ -0,0 +1,276 @@
package gouring
import (
"syscall"
"unsafe"
)
func (ring *IoUring) io_uring_register_buffers_update_tag(off uint32,
iov *syscall.Iovec,
tags []uint64,
nr uint32) error {
up := &IoUringRsrcUpdate2{
Offset: off,
Data: uint64(uintptr(unsafe.Pointer(iov))),
Tags: uint64(uintptr(unsafe.Pointer(&tags[0]))),
Nr: nr,
}
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_BUFFERS_UPDATE,
unsafe.Pointer(up), unsafe.Sizeof(*up))
if err != nil {
return err
}
_ = ret
return nil
}
func (ring *IoUring) io_uring_register_buffers_tags(
iov *syscall.Iovec,
tags []uint64,
nr uint32) error {
reg := &IoUringRsrcRegister{
Nr: nr,
Data: uint64(uintptr(unsafe.Pointer(iov))),
Tags: uint64(uintptr(unsafe.Pointer(&tags[0]))),
}
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_BUFFERS2,
unsafe.Pointer(reg), unsafe.Sizeof(*reg))
if err != nil {
return err
}
_ = ret
return nil
}
func (ring *IoUring) io_uring_register_buffers_sparse(nr uint32) error {
reg := &IoUringRsrcRegister{
Flags: IORING_RSRC_REGISTER_SPARSE,
Nr: nr,
}
ret, err := io_uring_register(ring.RingFd, IORING_RSRC_REGISTER_SPARSE,
unsafe.Pointer(reg), unsafe.Sizeof(*reg))
if err != nil {
return err
}
_ = ret
return nil
}
func (ring *IoUring) io_uring_register_buffers(iov *syscall.Iovec, nrIov uint32) int {
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_BUFFERS,
unsafe.Pointer(iov), uintptr(nrIov))
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_unregister_buffers() int {
ret, err := io_uring_register(ring.RingFd, IORING_UNREGISTER_BUFFERS, nil, 0)
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_register_files_update_tag(off uint32,
files []int, tags []uint64,
nrFiles uint32) (int, error) {
up := &IoUringRsrcUpdate2{
Offset: off,
Data: uint64(uintptr(unsafe.Pointer(&files[0]))),
Tags: uint64(uintptr(unsafe.Pointer(&tags[0]))),
Nr: nrFiles,
}
return io_uring_register(ring.RingFd, IORING_REGISTER_FILES_UPDATE2,
unsafe.Pointer(up),
unsafe.Sizeof(*up))
}
func (ring *IoUring) io_uring_register_files_update(off uint32,
files []int, nrFiles uint32) (int, error) {
up := &IoUringFilesUpdate{
Offset: off,
Fds: uint64(uintptr(unsafe.Pointer(&files[0]))),
}
return io_uring_register(ring.RingFd, IORING_REGISTER_FILES_UPDATE,
unsafe.Pointer(up), uintptr(nrFiles))
}
func (ring *IoUring) io_uring_register_files_sparse(nr uint32) (ret int, err error) {
reg := &IoUringRsrcRegister{
Flags: IORING_RSRC_REGISTER_SPARSE,
Nr: nr,
}
var didIncrease bool
for {
ret, err = io_uring_register(ring.RingFd, IORING_REGISTER_FILES2,
unsafe.Pointer(reg),
unsafe.Sizeof(*reg))
if err == nil {
break
}
if err == syscall.EMFILE && !didIncrease {
increase_rlimit_nofile(uint64(nr))
didIncrease = true
continue
}
break
}
return
}
func (ring *IoUring) io_uring_register_files_tags(
files []int,
tags []uint64, nr uint32) (ret int, err error) {
reg := &IoUringRsrcRegister{
Nr: nr,
Data: uint64(uintptr(unsafe.Pointer(&files[0]))),
Tags: uint64(uintptr(unsafe.Pointer(&tags[0]))),
}
var didIncrease bool
for {
ret, err = io_uring_register(ring.RingFd, IORING_REGISTER_FILES2,
unsafe.Pointer(reg), unsafe.Sizeof(*reg))
if err == nil {
break
}
if err == syscall.EMFILE && !didIncrease {
increase_rlimit_nofile(uint64(nr))
didIncrease = true
continue
}
break
}
return
}
func (ring *IoUring) io_uring_register_files(
files []int, nrFiles uint32) (ret int, err error) {
var didIncrease bool
for {
ret, err = io_uring_register(ring.RingFd, IORING_REGISTER_FILES,
unsafe.Pointer(&files[0]), uintptr(nrFiles))
if err == nil {
break
}
if err == syscall.EMFILE && !didIncrease {
increase_rlimit_nofile(uint64(nrFiles))
didIncrease = true
continue
}
break
}
return
}
func (ring *IoUring) io_uring_unregister_files() int {
ret, err := io_uring_register(ring.RingFd, IORING_UNREGISTER_FILES, nil, 0)
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_unregister_eventfd() int {
ret, err := io_uring_register(ring.RingFd, IORING_UNREGISTER_EVENTFD, nil, 0)
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_register_eventfd_async(eventFd int) int {
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_EVENTFD_ASYNC, nil, 0)
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_register_probe(p *IoUringProbe, nrOps uint32) int {
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_PROBE,
unsafe.Pointer(p), uintptr(nrOps))
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_register_personality() (int, error) {
return io_uring_register(ring.RingFd, IORING_REGISTER_PERSONALITY, nil, 0)
}
func (ring *IoUring) io_uring_unregister_personality(id int32) (int, error) {
return io_uring_register(ring.RingFd, IORING_UNREGISTER_PERSONALITY, nil, uintptr(id))
}
func (ring *IoUring) io_uring_register_restrictions(res *IoUringRestriction, nrRes uint32) int {
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_RESTRICTIONS,
unsafe.Pointer(res), uintptr(nrRes))
if err != nil {
return 0
}
return ret
}
func (ring *IoUring) io_uring_enable_rings() error {
_, err := io_uring_register(ring.RingFd, IORING_REGISTER_ENABLE_RINGS, nil, 0)
return err
}
// sched.h
// func io_uring_register_iowq_aff(ring *IoUring, cpuSz int, mask *CpuSet) {
// }
func (ring *IoUring) io_uring_unregister_iowq_aff() error {
_, err := io_uring_register(ring.RingFd, IORING_UNREGISTER_IOWQ_AFF, nil, 0)
return err
}
func (ring *IoUring) io_uring_register_iowq_max_workers(val *uint32) (int, error) {
return io_uring_register(ring.RingFd, IORING_REGISTER_IOWQ_MAX_WORKERS,
unsafe.Pointer(val), 2)
}
func (ring *IoUring) io_uring_register_ring_fd() (int, error) {
up := &IoUringRsrcUpdate{
Data: uint64(ring.RingFd),
Offset: ^uint32(0),
}
ret, err := io_uring_register(ring.RingFd, IORING_REGISTER_RING_FDS,
unsafe.Pointer(up), 1)
if err != nil {
return 0, err
}
ring.EnterRingFd = int(up.Offset)
ring.IntFlags |= INT_FLAG_REG_RING
return ret, nil
}
func (ring *IoUring) io_uring_unregister_ring_fd() error {
up := &IoUringRsrcUpdate{
Offset: uint32(ring.EnterRingFd),
}
ret, err := io_uring_register(ring.RingFd, IORING_UNREGISTER_RING_FDS,
unsafe.Pointer(up), 1)
if err != nil {
return err
}
if ret == 1 {
ring.EnterRingFd = ring.RingFd
ring.IntFlags &= ^INT_FLAG_REG_RING
}
return nil
}
func (ring *IoUring) io_uring_register_buf_ring(reg *IoUringBufReg, flags uint32) (int, error) {
return io_uring_register(ring.RingFd, IORING_REGISTER_PBUF_RING, unsafe.Pointer(reg), 1)
}
func (ring *IoUring) io_uring_unregister_buf_ring(bgId int32) (int, error) {
reg := &IoUringBufReg{
Bgid: uint16(bgId),
}
return io_uring_register(ring.RingFd, IORING_UNREGISTER_PBUF_RING, unsafe.Pointer(reg), 1)
}

131
ring.go
View file

@ -1,131 +0,0 @@
package gouring
import (
"sync/atomic"
"unsafe"
)
type Ring struct {
fd int
params IOUringParams
sq SQRing
cq CQRing
// cached ringn value
sqRingPtr, cqRingPtr, sqesPtr uintptr
sringSz, cringSz uint32
}
//
//-- SQ
type SQRing struct {
head uintptr
tail uintptr
ringMask uintptr
ringEntries uintptr
flags uintptr
array uint32Array
sqes sqeArray
}
func (sq SQRing) Get(idx uint32) *SQEntry {
if uintptr(idx) >= uintptr(*sq.RingEntries()) {
return nil
}
return sq.sqes.Get(uintptr(idx))
}
func (sq SQRing) Head() *uint32 {
return (*uint32)(unsafe.Pointer(sq.head))
}
func (sq SQRing) Tail() *uint32 {
return (*uint32)(unsafe.Pointer(sq.tail))
}
func (sq SQRing) RingMask() *uint32 {
return (*uint32)(unsafe.Pointer(sq.ringMask))
}
func (sq SQRing) RingEntries() *uint32 {
return (*uint32)(unsafe.Pointer(sq.ringEntries))
}
func (sq SQRing) Flags() *uint32 {
return (*uint32)(unsafe.Pointer(sq.flags))
}
func (sq SQRing) Array() uint32Array {
return sq.array
}
func (sq SQRing) Event() sqeArray {
return sq.sqes
}
func (sq SQRing) IsCQOverflow() bool {
return atomic.LoadUint32(sq.Flags())&IORING_SQ_CQ_OVERFLOW > 0
}
func (sq SQRing) IsNeedWakeup() bool {
return atomic.LoadUint32(sq.Flags())&IORING_SQ_NEED_WAKEUP > 0
}
//
type uint32Array uintptr
func (a uint32Array) Get(idx uint32) *uint32 {
return (*uint32)(unsafe.Pointer(uintptr(a) + uintptr(idx)*_sz_uint32))
}
func (a uint32Array) Set(idx uint32, v uint32) {
atomic.StoreUint32(a.Get(idx), v)
}
type sqeArray uintptr
func (sa sqeArray) Get(idx uintptr) *SQEntry {
return (*SQEntry)(unsafe.Pointer(uintptr(sa) + idx*_sz_sqe))
}
func (sa sqeArray) Set(idx uintptr, v SQEntry) {
*sa.Get(idx) = v
}
//
//-- CQ
type CQRing struct {
head uintptr
tail uintptr
ringMask uintptr
ringEntries uintptr
cqes cqeArray
}
func (cq CQRing) Get(idx uint32) *CQEntry {
if uintptr(idx) >= uintptr(*cq.RingEntries()) { // avoid lookup overflow
return nil
}
return cq.cqes.Get(uintptr(idx))
}
func (cq CQRing) Head() *uint32 {
return (*uint32)(unsafe.Pointer(cq.head))
}
func (cq CQRing) Tail() *uint32 {
return (*uint32)(unsafe.Pointer(cq.tail))
}
func (cq CQRing) RingMask() *uint32 {
return (*uint32)(unsafe.Pointer(cq.ringMask))
}
func (cq CQRing) RingEntries() *uint32 {
return (*uint32)(unsafe.Pointer(cq.ringEntries))
}
func (cq CQRing) Event() cqeArray {
return cq.cqes
}
//
type cqeArray uintptr
func (ca cqeArray) Get(idx uintptr) *CQEntry {
return (*CQEntry)(unsafe.Pointer(uintptr(ca) + idx*_sz_cqe))
}
func (ca cqeArray) Set(idx uintptr, v CQEntry) {
*ca.Get(idx) = v
}

103
ring_entry.go
View file

@ -1,103 +0,0 @@
package gouring
import (
"reflect"
"unsafe"
)
var (
_sqe = SQEntry{}
_sqe_mm = make([]byte, _sz_sqe)
_sz_sqe = unsafe.Sizeof(_sqe)
_cqe = CQEntry{}
_cqe_mm = make([]byte, _sz_cqe)
_sz_cqe = unsafe.Sizeof(_cqe)
)
//-- SQEntry
type SQEntry struct {
Opcode UringOpcode
Flags UringSQEFlag
Ioprio uint16
Fd int32
off__addr2 uint64 // union { off, addr2 }
addr__splice_off_in uint64 // union { addr, splice_off_in }
Len uint32
opcode__flags_events uint32 // union of events and flags for opcode
UserData uint64
buf__index_group uint16 // union {buf_index, buf_group}
Personality uint16
splice_fd_in__file_index int32 // union { __s32 splice_fd_in, __u32 file_index }
pad2 [2]uint64
}
func (sqe *SQEntry) Offset() *uint64 {
return &sqe.off__addr2
}
func (sqe *SQEntry) SetOffset(v uint64) {
*sqe.Offset() = v
}
func (sqe *SQEntry) Addr2() *uint64 {
return &sqe.off__addr2
}
func (sqe *SQEntry) SetAddr2(v interface{}) {
*sqe.Addr2() = (uint64)(reflect.ValueOf(v).Pointer())
}
func (sqe *SQEntry) Addr() *uint64 {
return &sqe.addr__splice_off_in
}
func (sqe *SQEntry) SetAddr(v interface{}) {
*sqe.Addr() = (uint64)(reflect.ValueOf(v).Pointer())
}
func (sqe *SQEntry) SpliceOffIn() *uint64 {
return &sqe.addr__splice_off_in
}
func (sqe *SQEntry) OpcodeFlags() *uint32 {
return &sqe.opcode__flags_events
}
func (sqe *SQEntry) OpodeEvents() *uint32 {
return &sqe.opcode__flags_events
}
func (sqe *SQEntry) BufIndex() *uint16 {
return &sqe.buf__index_group
}
func (sqe *SQEntry) BufGroup() *uint16 {
return &sqe.buf__index_group
}
func (sqe *SQEntry) SpliceFdIn() *int32 {
return &sqe.splice_fd_in__file_index
}
func (sqe *SQEntry) FileIndex() *uint32 {
return (*uint32)(unsafe.Pointer(&sqe.splice_fd_in__file_index))
}
//
func (sqe *SQEntry) Reset() {
*sqe = _sqe
}
//-- CQEntry
type CQEntry struct {
UserData uint64 /* sqe->data submission passed back */
Res int32 /* result code for this event */
Flags UringCQEFlag
}
func (cqe *CQEntry) Reset() {
*cqe = _cqe
}

140
setup.go Normal file
View file

@ -0,0 +1,140 @@
package gouring
import (
"syscall"
"unsafe"
)
// func io_uring_queue_init(entries uint32, ring *IoUring, flags uint32) error {
// p := new(IoUringParams)
// p.Flags = flags
// return io_uring_queue_init_params(entries, ring, p)
// }
func io_uring_queue_init_params(entries uint32, ring *IoUring, p *IoUringParams) error {
fd, err := io_uring_setup(uintptr(entries), p)
if err != nil {
return err
}
err = io_uring_queue_mmap(fd, p, ring)
if err != nil {
return err
}
ring.Features = p.Features
return nil
}
func (ring *IoUring) io_uring_queue_exit() {
sq := &ring.Sq
cq := &ring.Cq
sqeSize := SizeofIoUringSqe
if ring.Flags&IORING_SETUP_SQE128 != 0 {
sqeSize += 64
}
munmap(unsafe.Pointer(sq.Sqes), sqeSize*uintptr(*sq._RingEntries()))
io_uring_unmap_rings(sq, cq)
/*
* Not strictly required, but frees up the slot we used now rather
* than at process exit time.
*/
if ring.IntFlags&INT_FLAG_REG_RING != 0 {
ring.io_uring_unregister_ring_fd()
}
syscall.Close(int(ring.RingFd))
}
func io_uring_queue_mmap(fd int, p *IoUringParams, ring *IoUring) error {
err := io_uring_mmap(fd, p, &ring.Sq, &ring.Cq)
if err != nil {
return err
}
ring.Flags = p.Flags
ring.RingFd, ring.EnterRingFd = fd, fd
ring.IntFlags = 0
return nil
}
func io_uring_mmap(fd int, p *IoUringParams, sq *IoUringSq, cq *IoUringCq) (err error) {
size := SizeofIoUringCqe
if p.Flags&IORING_SETUP_CQE32 != 0 {
size += SizeofIoUringCqe
}
sq.RingSz = p.SqOff.Array + p.SqEntries*uint32(SizeofUnsigned)
cq.RingSz = p.CqOff.Cqes + p.CqEntries*uint32(size)
if p.Features&IORING_FEAT_SINGLE_MMAP != 0 {
if cq.RingSz > sq.RingSz {
sq.RingSz = cq.RingSz
}
// cq.RingSz = sq.RingSz
}
// alloc sq ring
sq.RingPtr, err = mmap(nil, uintptr(sq.RingSz),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
fd, IORING_OFF_SQ_RING)
if err != nil {
return
}
if p.Features&IORING_FEAT_SINGLE_MMAP != 0 {
cq.RingPtr = sq.RingPtr
} else {
// alloc cq ring
cq.RingPtr, err = mmap(nil, uintptr(cq.RingSz),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
fd, IORING_OFF_CQ_RING)
if err != nil {
// goto errLabel
io_uring_unmap_rings(sq, cq)
return
}
}
//sq
sq.head = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.Head)))
sq.tail = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.Tail)))
sq.ringMask = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.RingMask)))
sq.ringEntries = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.RingEntries)))
sq.flags = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.Flags)))
sq.dropped = (unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.Dropped)))
sq.Array = (uint32Array)(unsafe.Pointer(uintptr(sq.RingPtr) + uintptr(p.SqOff.Array)))
size = SizeofIoUringSqe
if p.Flags&IORING_SETUP_SQE128 != 0 {
size += 64
}
var sqeAddr unsafe.Pointer
sqeAddr, err = mmap(nil, size*uintptr(p.SqEntries),
syscall.PROT_READ|syscall.PROT_WRITE,
syscall.MAP_SHARED|syscall.MAP_POPULATE,
fd, IORING_OFF_SQES)
if err != nil {
//errLabel:
io_uring_unmap_rings(sq, cq)
return
}
sq.Sqes = (ioUringSqeArray)(sqeAddr)
//cq
cq.head = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.Head)))
cq.tail = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.Tail)))
cq.ringMask = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.RingMask)))
cq.ringEntries = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.RingEntries)))
cq.overflow = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.Overflow)))
cq.Cqes = (ioUringCqeArray)(unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.Cqes)))
if p.CqOff.Flags != 0 {
cq.flags = (unsafe.Pointer(uintptr(cq.RingPtr) + uintptr(p.CqOff.Flags)))
}
return nil
}
func io_uring_unmap_rings(sq *IoUringSq, cq *IoUringCq) error {
munmap(sq.RingPtr, uintptr(sq.RingSz))
if cq.RingPtr != nil && cq.RingPtr != sq.RingPtr {
munmap(cq.RingPtr, uintptr(cq.RingSz))
}
return nil
}

11
sigset.go
View file

@ -5,12 +5,11 @@ import (
)
const (
_uint64 uint64 = 0
_sz_uint64 = unsafe.Sizeof(_uint64)
SIGSET_NWORDS = (1024 / (8 * _sz_uint64))
SIGTMIN = 32
SIGTMAX = SIGTMIN
NSIG = (SIGTMAX + 1)
SizeofUint64 = unsafe.Sizeof(uint64(0))
SIGSET_NWORDS = (1024 / (8 * SizeofUint64))
SIGTMIN = 32
SIGTMAX = SIGTMIN
NSIG = (SIGTMAX + 1)
)
type Sigset_t struct {

63
syscall.go Normal file
View file

@ -0,0 +1,63 @@
package gouring
import (
"syscall"
"unsafe"
)
func io_uring_setup(entries uintptr, params *IoUringParams) (ret int, err error) {
r1, _, e1 := syscall.Syscall(SYS_IO_URING_SETUP, entries, uintptr(unsafe.Pointer(params)), 0)
ret = int(r1)
if e1 < 0 {
err = e1
}
return
}
func io_uring_enter(fd int, toSubmit uint32, minComplete uint32, flags uint32, sig *Sigset_t) (ret int, err error) {
return io_uring_enter2(fd, toSubmit, minComplete, flags, sig, NSIG/8)
}
// TODO: decide to use Syscall or RawSyscall
func io_uring_enter2(fd int, toSubmit uint32, minComplete uint32, flags uint32, sig *Sigset_t, sz int32) (ret int, err error) {
r1, _, e1 := syscall.Syscall6(SYS_IO_URING_ENTER,
uintptr(fd),
uintptr(toSubmit), uintptr(minComplete),
uintptr(flags), uintptr(unsafe.Pointer(sig)), uintptr(sz))
ret = int(r1)
if e1 != 0 {
err = e1
}
return
}
func io_uring_register(fd int, opcode uint32, arg unsafe.Pointer, nrArgs uintptr) (ret int, err error) {
r1, _, e1 := syscall.Syscall6(SYS_IO_URING_REGISTER, uintptr(fd), uintptr(opcode), uintptr(arg), uintptr(nrArgs), 0, 0)
ret = int(r1)
if e1 != 0 {
err = e1
}
return
}
//go:linkname mmap syscall.mmap
func mmap(addr unsafe.Pointer, length uintptr, prot int, flags int, fd int, offset int64) (xaddr unsafe.Pointer, err error)
//go:linkname munmap syscall.munmap
func munmap(addr unsafe.Pointer, length uintptr) (err error)
//
func increase_rlimit_nofile(nr uint64) error {
var rlim syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
return err
}
if rlim.Cur < nr {
rlim.Cur += nr
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
}
return err
}

9
syscall_nr_amd64.go Normal file
View file

@ -0,0 +1,9 @@
package gouring
const (
// uring syscall no.
SYS_IO_URING_SETUP = 425
SYS_IO_URING_ENTER = 426
SYS_IO_URING_REGISTER = 427
)

86
sysnum.go
View file

@ -1,86 +0,0 @@
package gouring
import (
"syscall"
"unsafe"
)
const (
SYS_IO_URING_SETUP = 425
SYS_IO_URING_ENTER = 426
SYS_IO_URING_REGISTER = 427
)
//go:linkname errnoErr syscall.errnoErr
func errnoErr(e syscall.Errno) error
type SQOffsets struct {
Head uint32
Tail uint32
RingMask uint32
RingEntries uint32
Flags uint32
Dropped uint32
Array uint32
Resv1 uint32
Resv2 uint64
}
type CQOffsets struct {
Head uint32
Tail uint32
RingMask uint32
RingEntries uint32
Overflow uint32
CQEs uint32
Flags uint32
Resv1 uint32
Resv2 uint64
}
type IOUringParams struct {
SQEntries uint32 // sq_entries
CQEntries uint32 // cq_entries
Flags UringSetupFlag // flags
SQThreadCPU uint32 // sq_thread_cpu
SQThreadIdle uint32 // sq_threead_idle
Features UringParamFeatureFlag // features
WQFd uint32 // wq_fd
resv [3]uint32 // resv
SQOff SQOffsets // sq_off
CQOff CQOffsets // cq_off
}
//go:inline
func io_uring_setup(entries uint, params *IOUringParams) (fd int, err error) {
r1, _, e1 := syscall.Syscall(SYS_IO_URING_SETUP, uintptr(entries), uintptr(unsafe.Pointer(params)), 0)
fd = int(r1)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
func io_uring_enter(ringFd int, toSubmit uint, minComplete uint, flags uint, sig *Sigset_t) (ret int, err error) {
return io_uring_enter2(ringFd, toSubmit, minComplete, flags, sig, NSIG/8)
}
func io_uring_enter2(ringFd int, toSubmit uint, minComplete uint, flags uint, sig *Sigset_t, sz int) (ret int, err error) {
r1, _, e1 := syscall.Syscall6(SYS_IO_URING_ENTER, uintptr(ringFd), uintptr(toSubmit), uintptr(minComplete), uintptr(flags), uintptr(unsafe.Pointer(sig)), uintptr(sz))
ret = int(r1)
if e1 != 0 {
err = errnoErr(e1)
}
return
}
func io_uring_register(ringFd int, opcode uint /*const*/, arg uintptr, nrArgs uint) (ret int, err error) {
r1, _, e1 := syscall.Syscall6(SYS_IO_URING_REGISTER, uintptr(ringFd), uintptr(opcode), arg, uintptr(nrArgs), 0, 0)
ret = int(r1)
if e1 != 0 {
err = errnoErr(e1)
}
return
}

48
uring.go Normal file
View file

@ -0,0 +1,48 @@
package gouring
func New(entries uint32, flags uint32) (*IoUring, error) {
ring := &IoUring{}
p := new(IoUringParams)
p.Flags = flags
err := io_uring_queue_init_params(entries, ring, p)
if err != nil {
return nil, err
}
return ring, nil
}
func NewWithParams(entries uint32, params *IoUringParams) (*IoUring, error) {
ring := &IoUring{}
if params == nil {
params = new(IoUringParams)
}
err := io_uring_queue_init_params(entries, ring, params)
if err != nil {
return nil, err
}
return ring, nil
}
func (h *IoUring) Close() {
h.io_uring_queue_exit()
}
func (h *IoUring) GetSqe() *IoUringSqe {
return h.io_uring_get_sqe()
}
func (h *IoUring) WaitCqe(cqePtr **IoUringCqe) error {
return h.io_uring_wait_cqe(cqePtr)
}
func (h *IoUring) SeenCqe(cqe *IoUringCqe) {
h.io_uring_cqe_seen(cqe)
}
func (h *IoUring) Submit() (int, error) {
return h.io_uring_submit()
}
func (h *IoUring) SubmitAndWait(waitNr uint32) (int, error) {
return h.io_uring_submit_and_wait(waitNr)
}

81
uring_bench_test.go Normal file
View file

@ -0,0 +1,81 @@
package gouring
import (
"context"
"runtime"
"syscall"
"testing"
)
func BenchmarkQueueNop(b *testing.B) {
type opt struct {
name string
entries uint32
p IoUringParams
}
ts := []opt{
{"def-256", 256, IoUringParams{Flags: 0}},
{"sqpoll-256-4-10000", 256, IoUringParams{Flags: IORING_SETUP_SQPOLL, SqThreadCpu: 16, SqThreadIdle: 10_000}},
}
consumer := func(h *IoUring, ctx context.Context, count int) {
var cqe *IoUringCqe
var err error
for i := 0; i < count; i++ {
if ctx.Err() != nil {
return
}
err = h.WaitCqe(&cqe)
if err == syscall.EINTR {
continue // ignore INTR
} else if err != nil {
panic(err)
}
if cqe.Res < 0 {
panic(syscall.Errno(-cqe.Res))
}
h.SeenCqe(cqe)
}
}
for _, tc := range ts {
b.Run(tc.name, func(b *testing.B) {
h := testNewIoUringWithParams(b, tc.entries, &tc.p)
defer h.Close()
var (
j uint32
sqe *IoUringSqe
err error
submitted int
)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
b.ResetTimer()
for i := 0; i < b.N; i++ {
for j = 0; j < tc.entries; j++ {
for {
// sqe could be nil if SQ is already full so we spin until we got one
sqe = h.GetSqe()
if sqe != nil {
break
}
runtime.Gosched()
}
PrepNop(sqe)
sqe.UserData.SetUint64(uint64(i + int(j)))
}
submitted, err = h.Submit()
if err != nil {
panic(err)
}
consumer(h, ctx, submitted)
}
b.StopTimer()
})
}
}

64
uring_test.go Normal file
View file

@ -0,0 +1,64 @@
package gouring
import (
"bytes"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type genericTestingT interface {
assert.TestingT
require.TestingT
}
func testNewIoUring(t genericTestingT, entries uint32, flags uint32) *IoUring {
h, err := New(entries, flags)
require.NoError(t, err)
require.NotNil(t, h)
return h
}
func testNewIoUringWithParams(t genericTestingT, entries uint32, p *IoUringParams) *IoUring {
h, err := NewWithParams(entries, p)
require.NoError(t, err)
require.NotNil(t, h)
return h
}
func TestRingWrapper(t *testing.T) {
h := testNewIoUring(t, 256, 0)
defer h.Close()
// O_RDWR|O_CREATE|O_EXCL
ftmp, err := os.CreateTemp(os.TempDir(), "test_iouring_ring_wrapper_*")
require.NoError(t, err)
fd := ftmp.Fd()
var whatToWrite = [][]byte{
[]byte("hello\n"),
[]byte("\tworld\n\n"),
[]byte("io_uring\t\t\n"),
[]byte("nice!\n!!!\n\x00"),
}
var off uint64 = 0
for _, bs := range whatToWrite {
sqe := h.GetSqe()
PrepWrite(sqe, int(fd), &bs[0], len(bs), off)
sqe.Flags = IOSQE_IO_LINK
off = off + uint64(len(bs))
}
submitted, err := h.SubmitAndWait(uint32(len(whatToWrite)))
require.NoError(t, err)
require.Equal(t, len(whatToWrite), int(submitted))
var readed = make([]byte, 1024)
nb, err := ftmp.Read(readed)
assert.NoError(t, err)
readed = readed[:nb]
joined := bytes.Join(whatToWrite, []byte{})
assert.Equal(t, joined, readed)
}

96
util_ptr_arith.go Normal file
View file

@ -0,0 +1,96 @@
package gouring
import (
"unsafe"
)
type uint32Array = unsafe.Pointer // *uint32
func uint32Array_Index(u uint32Array, i uintptr) *uint32 {
return (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(u)) + SizeofUint32*i))
}
type ioUringSqeArray = unsafe.Pointer // *IoUringSqe
func ioUringSqeArray_Index(u ioUringSqeArray, i uintptr) *IoUringSqe {
return (*IoUringSqe)(unsafe.Pointer(uintptr(unsafe.Pointer(u)) + SizeofIoUringSqe*i))
}
func ioUringSqe128Array_Index(u ioUringSqeArray, i uintptr) *IoUringSqe {
return (*IoUringSqe)(unsafe.Pointer(uintptr(unsafe.Pointer(u)) + (SizeofIoUringSqe+64)*i))
}
//
type ioUringCqeArray = unsafe.Pointer // *IoUringCqe
func ioUringCqeArray_Index(u ioUringCqeArray, i uintptr) *IoUringCqe {
return (*IoUringCqe)(unsafe.Pointer(uintptr(unsafe.Pointer(u)) + SizeofIoUringCqe*i))
}
func ioUringCqe32Array_Index(u ioUringCqeArray, i uintptr) *IoUringCqe {
return (*IoUringCqe)(unsafe.Pointer(uintptr(unsafe.Pointer(u)) + (SizeofIoUringCqe+SizeofIoUringCqe)*i))
}
//
type UserData [8]byte // uint64
func (u *UserData) SetUint64(v uint64) {
putUintptr(unsafe.Pointer(u), uintptr(v))
}
func (u *UserData) SetUintptr(v uintptr) {
putUintptr(unsafe.Pointer(u), v)
}
func (u *UserData) SetUnsafe(ptr unsafe.Pointer) {
putUnsafe(unsafe.Pointer(u), ptr)
}
func (u UserData) GetUnsafe() unsafe.Pointer {
return *(*unsafe.Pointer)(unsafe.Pointer(&u))
}
func (u UserData) GetUintptr() uintptr {
return uintptr(u.GetUnsafe())
}
func (u UserData) GetUint64() uint64 {
return uint64(u.GetUintptr())
}
func (u UserData) IsZero() bool {
return u.GetUintptr() == 0
}
// ---
func putUnsafe(ptr unsafe.Pointer, v unsafe.Pointer) {
*(*unsafe.Pointer)(ptr) = v
}
func putUintptr(ptr unsafe.Pointer, v uintptr) {
*(*uintptr)(ptr) = v
}
func putUint64(ptr unsafe.Pointer, v uint64) {
*(*uint64)(ptr) = v
}
func putUint32(ptr unsafe.Pointer, v uint32) {
*(*uint32)(ptr) = v
}
func putUint16(ptr unsafe.Pointer, v uint16) {
*(*uint16)(ptr) = v
}
func putUint8(ptr unsafe.Pointer, v uint8) {
*(*uint8)(ptr) = v
}
func putInt64(ptr unsafe.Pointer, v int64) {
*(*int64)(ptr) = v
}
func putInt32(ptr unsafe.Pointer, v int32) {
*(*int32)(ptr) = v
}
func putInt16(ptr unsafe.Pointer, v int16) {
*(*int16)(ptr) = v
}
func putInt8(ptr unsafe.Pointer, v int8) {
*(*int8)(ptr) = v
}

44
util_ptr_arith_test.go Normal file
View file

@ -0,0 +1,44 @@
package gouring
import (
"encoding/binary"
"fmt"
"testing"
"unsafe"
"github.com/stretchr/testify/assert"
)
func TestUserdata(t *testing.T) {
type test struct {
v any
exp uint64
}
ts := []test{
{uint64(0), 0},
{uint64(0xff), 0xff},
{uint64(0xfffefd), 0xfffefd},
{uintptr(0xcafeba), 0xcafeba},
{unsafe.Pointer(nil), 0},
}
bo := binary.LittleEndian
for _, tc := range ts {
var u UserData
switch v := tc.v.(type) {
case uint64:
u.SetUint64(v)
case uintptr:
u.SetUintptr(v)
case unsafe.Pointer:
u.SetUnsafe(v)
default:
panic(fmt.Sprintf("unhandled type: %T", v))
}
assert.Equal(t, tc.exp, u.GetUint64())
var exp [8]byte
bo.PutUint64(exp[:], tc.exp)
assert.Equal(t, exp[:], u[:])
}
}

52
util_union.go Normal file
View file

@ -0,0 +1,52 @@
package gouring
import "unsafe"
type union64 [8]byte
func (u *union64) PutUnsafe(v unsafe.Pointer) { putUnsafe(unsafe.Pointer(u), v) }
func (u *union64) PutUintptr(v uintptr) { putUintptr(unsafe.Pointer(u), v) }
func (u *union64) PutUint64(v uint64) { putUint64(unsafe.Pointer(u), v) }
func (u *union64) PutUint32(v uint32) { putUint32(unsafe.Pointer(u), v) }
func (u *union64) PutUint16(v uint16) { putUint16(unsafe.Pointer(u), v) }
func (u *union64) PutUint8(v uint8) { putUint8(unsafe.Pointer(u), v) }
func (u *union64) PutInt32(v int32) { putInt32(unsafe.Pointer(u), v) }
func (u *union64) Unsafe() unsafe.Pointer { return unsafe.Pointer(u) }
func (u *union64) Uint64() uint64 { return *(*uint64)(unsafe.Pointer(u)) }
func (u *union64) Uint32() uint32 { return *(*uint32)(unsafe.Pointer(u)) }
func (u *union64) Uint16() uint16 { return *(*uint16)(unsafe.Pointer(u)) }
func (u *union64) Uint8() uint8 { return *(*uint8)(unsafe.Pointer(u)) }
type union32 [4]byte
func (u *union32) PutUnsafe(v unsafe.Pointer) { putUnsafe(unsafe.Pointer(u), v) }
func (u *union32) PutUintptr(v uintptr) { putUintptr(unsafe.Pointer(u), uintptr(uint32(v))) }
func (u *union32) PutUint64(v uint64) { putUint32(unsafe.Pointer(u), uint32(v)) }
func (u *union32) PutUint32(v uint32) { putUint32(unsafe.Pointer(u), v) }
func (u *union32) PutUint16(v uint16) { putUint16(unsafe.Pointer(u), v) }
func (u *union32) PutUint8(v uint8) { putUint8(unsafe.Pointer(u), v) }
func (u *union32) PutInt32(v int32) { putInt32(unsafe.Pointer(u), v) }
func (u *union32) Unsafe() unsafe.Pointer { return unsafe.Pointer(u) }
func (u *union32) Uint64() uint64 { return *(*uint64)(unsafe.Pointer(u)) }
func (u *union32) Uint32() uint32 { return *(*uint32)(unsafe.Pointer(u)) }
func (u *union32) Uint16() uint16 { return *(*uint16)(unsafe.Pointer(u)) }
func (u *union32) Uint8() uint8 { return *(*uint8)(unsafe.Pointer(u)) }
type union16 [2]byte
func (u *union16) PutUnsafe(v unsafe.Pointer) { putUnsafe(unsafe.Pointer(u), v) }
func (u *union16) PutUintptr(v uintptr) { putUintptr(unsafe.Pointer(u), uintptr(uint16(v))) }
func (u *union16) PutUint64(v uint64) { putUint16(unsafe.Pointer(u), uint16(v)) }
func (u *union16) PutUint32(v uint32) { putUint16(unsafe.Pointer(u), uint16(v)) }
func (u *union16) PutUint16(v uint16) { putUint16(unsafe.Pointer(u), v) }
func (u *union16) PutUint8(v uint8) { putUint8(unsafe.Pointer(u), v) }
func (u *union16) Unsafe() unsafe.Pointer { return unsafe.Pointer(u) }
func (u *union16) Uint64() uint64 { return *(*uint64)(unsafe.Pointer(u)) }
func (u *union16) Uint32() uint32 { return *(*uint32)(unsafe.Pointer(u)) }
func (u *union16) Uint16() uint16 { return *(*uint16)(unsafe.Pointer(u)) }
func (u *union16) Uint8() uint8 { return *(*uint8)(unsafe.Pointer(u)) }