Compare commits

..

No commits in common. "main" and "0.5.0" have entirely different histories.
main ... 0.5.0

72 changed files with 582 additions and 1516 deletions

1
.github/FUNDING.yml vendored
View File

@ -1 +0,0 @@
github: [nalgeon]

View File

@ -7,12 +7,6 @@ on:
- "docs/**"
- Makefile
- README.md
pull_request:
branches: [main]
paths-ignore:
- "docs/**"
- Makefile
- README.md
workflow_dispatch:
defaults:
@ -24,10 +18,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@v4
with:
go-version: "stable"
@ -35,8 +29,13 @@ jobs:
run: make test build
- name: Upload artifact
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: codapi
path: build/codapi
path: |
build/codapi
images/
*.json
codapi.service
Makefile
retention-days: 7

View File

@ -14,15 +14,15 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@v4
with:
go-version: "stable"
- name: Release and publish
uses: goreleaser/goreleaser-action@v5
uses: goreleaser/goreleaser-action@v4
with:
args: release --clean
env:

View File

@ -11,7 +11,7 @@ builds:
archives:
- files:
- configs/*
- "*.json"
- images/*
- codapi.service
- LICENSE

View File

@ -6,7 +6,9 @@ build_rev := "main"
ifneq ($(wildcard .git),)
build_rev := $(shell git rev-parse --short HEAD)
endif
build_date := $(shell date -u '+%Y-%m-%dT%H:%M:%S')
build_date := $(shell date -u '+%Y%m%d')
version := $(build_date):$(build_rev)
setup:
@go mod download
@ -22,7 +24,7 @@ test:
build:
@go build -ldflags "-X main.commit=$(build_rev) -X main.date=$(build_date)" -o build/codapi -v cmd/main.go
@go build -ldflags "-X main.Version=$(version)" -o build/codapi -v cmd/main.go
run:
@./build/codapi

View File

@ -1,17 +1,19 @@
# Interactive code examples
# Embeddable code playgrounds
_for documentation, education and fun_ 🎉
_for education, documentation, and fun_ 🎉
Codapi is a platform for embedding interactive code snippets directly into your product documentation, online course or blog post.
Codapi is a platform for embedding interactive code snippets directly into your product documentation, online course, or blog post.
```
python
┌───────────────────────────────┐
def greet(name):
print(f"Hello, {name}!")
msg = "Hello, World!"
print(msg)
│ │
│ greet("World") │
│ │
│ run ► │
└───────────────────────────────┘
Run ► Edit ✓ Done
✓ Done
┌───────────────────────────────┐
│ Hello, World! │
└───────────────────────────────┘
@ -21,35 +23,75 @@ Codapi manages sandboxes (isolated execution environments) and provides an API t
Highlights:
- Automatically converts static code examples into mini-playgrounds.
- Lightweight and easy to integrate.
- Sandboxes for any programming language, database, or software.
- Supports dozens of playgrounds out of the box, plus custom sandboxes if you need them.
- Available as a cloud service and as a self-hosted version.
- Open source. Uses the permissive Apache-2.0 license.
- Lightweight and easy to integrate.
For an introduction to Codapi, see this post: [Interactive code examples for fun and profit](https://antonz.org/code-examples/).
Learn more at [**codapi.org**](https://codapi.org/)
## Installation
See [Installing Codapi](docs/install.md) for details.
## Usage
## Usage (API)
See [API](docs/api.md) to run sandboxed code using the HTTP API.
Call `/v1/exec` to run the code in a sandbox:
See [codapi-js](https://github.com/nalgeon/codapi-js) to embed the JavaScript widget into a web page.
```http
POST https://api.codapi.org/v1/exec
content-type: application/json
{
"sandbox": "python",
"command": "run",
"files": {
"": "print('hello world')"
}
}
```
`sandbox` is the name of the pre-configured sandbox, and `command` is the name of a command supported by that sandbox. See [Configuration](docs/config.md) for details.
`files` is a map, where the key is a filename and the value is its contents. When executing a single file, it should either be named as the `command` expects, or be an empty string (as in the example above).
Response:
```http
HTTP/1.1 200 OK
Content-Type: application/json
{
"id": "python_run_9b7b1afd",
"ok": true,
"duration": 314,
"stdout": "hello world\n",
"stderr": ""
}
```
- `id` is the unique execution identifier.
- `ok` is `true` if the code executed without errors, or `false` otherwise.
- `duration` is the execution time in milliseconds.
- `stdout` is what the code printed to the standard output.
- `stderr` is what the code printed to the standard error, or a compiler/os error (if any).
## Usage (JavaScript)
See [codapi-js](https://github.com/nalgeon/codapi-js) to embed the JavaScript widget into a web page. The widget uses exactly the same API as described above.
## Contributing
Contributions are welcome. For anything other than bugfixes, please first open an issue to discuss what you want to change.
Pull requests are welcome. For new features and improvements, please first open an issue to discuss what you would like to change.
Be sure to add or update tests as appropriate.
## Funding
## License
Codapi is mostly a [one-man](https://antonz.org/) project, not backed by a VC fund or anything.
Copyright 2023 [Anton Zhiyanov](https://antonz.org/).
If you find Codapi useful, please consider sponsoring it on GitHub. It really helps to move the project forward.
The software is available under the Apache-2.0 license.
♥ [Become a sponsor](https://github.com/sponsors/nalgeon) to support Codapi.
## Stay tuned
★ [Subscribe](https://antonz.org/subscribe/) to stay on top of new features.
★ [**Subscribe**](https://antonz.org/subscribe/) to stay on top of new features.

View File

@ -7,22 +7,17 @@ import (
"os/signal"
"syscall"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/internal/sandbox"
"github.com/nalgeon/codapi/internal/server"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/logx"
"github.com/nalgeon/codapi/sandbox"
"github.com/nalgeon/codapi/server"
)
// set by the build process
var (
version = "main"
commit = "none"
date = "unknown"
)
var Version string = "main"
// startServer starts the HTTP API sandbox server.
func startServer(port int) *server.Server {
logx.Log("codapi %s, commit %s, built at %s", version, commit, date)
logx.Log("codapi %s", Version)
logx.Log("listening on port %d...", port)
router := server.NewRouter()
srv := server.NewServer(port, router)
@ -47,7 +42,7 @@ func main() {
port := flag.Int("port", 1313, "server port")
flag.Parse()
cfg, err := config.Read("configs")
cfg, err := config.Read("config.json", "boxes.json", "commands.json")
if err != nil {
logx.Log("missing config file")
os.Exit(1)

14
commands.json Normal file
View File

@ -0,0 +1,14 @@
{
"sh": {
"run": {
"engine": "docker",
"entry": "main.sh",
"steps": [
{
"box": "alpine",
"command": ["sh", "main.sh"]
}
]
}
}
}

View File

@ -6,7 +6,7 @@ import (
"sort"
)
// A Config describes application config.
// A Config describes application cofig.
type Config struct {
PoolSize int `json:"pool_size"`
Verbose bool `json:"verbose"`
@ -96,10 +96,8 @@ type Command struct {
// A Step describes a single step of a command.
type Step struct {
Box string `json:"box"`
Version string `json:"version"`
User string `json:"user"`
Action string `json:"action"`
Detach bool `json:"detach"`
Stdin bool `json:"stdin"`
Command []string `json:"command"`
Timeout int `json:"timeout"`

94
config/load.go Normal file
View File

@ -0,0 +1,94 @@
package config
import (
"encoding/json"
"os"
)
// Read reads application config from JSON files.
func Read(cfgPath, boxPath, cmdPath string) (*Config, error) {
cfg, err := ReadConfig(cfgPath)
if err != nil {
return nil, err
}
cfg, err = ReadBoxes(cfg, boxPath)
if err != nil {
return nil, err
}
cfg, err = ReadCommands(cfg, cmdPath)
if err != nil {
return nil, err
}
return cfg, err
}
// ReadConfig reads application config from a JSON file.
func ReadConfig(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
cfg := &Config{}
err = json.Unmarshal(data, cfg)
if err != nil {
return nil, err
}
return cfg, err
}
// ReadBoxes reads boxes config from a JSON file.
func ReadBoxes(cfg *Config, path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
boxes := make(map[string]*Box)
err = json.Unmarshal(data, &boxes)
if err != nil {
return nil, err
}
for _, box := range boxes {
setBoxDefaults(box, cfg.Box)
}
cfg.Boxes = boxes
return cfg, err
}
// ReadCommands reads commands config from a JSON file.
func ReadCommands(cfg *Config, path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
commands := make(map[string]SandboxCommands)
err = json.Unmarshal(data, &commands)
if err != nil {
return nil, err
}
for _, playCmds := range commands {
for _, cmd := range playCmds {
if cmd.Before != nil {
setStepDefaults(cmd.Before, cfg.Step)
}
for _, step := range cmd.Steps {
setStepDefaults(step, cfg.Step)
}
if cmd.After != nil {
setStepDefaults(cmd.After, cfg.Step)
}
}
}
cfg.Commands = commands
return cfg, err
}

View File

@ -1,11 +1,15 @@
package config
import (
"path/filepath"
"testing"
)
func TestRead(t *testing.T) {
cfg, err := Read("testdata")
cfgPath := filepath.Join("testdata", "config.json")
boxPath := filepath.Join("testdata", "boxes.json")
cmdPath := filepath.Join("testdata", "commands.json")
cfg, err := Read(cfgPath, boxPath, cmdPath)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

25
config/testdata/commands.json vendored Normal file
View File

@ -0,0 +1,25 @@
{
"python": {
"run": {
"engine": "docker",
"entry": "main.py",
"steps": [
{
"box": "python",
"command": ["python", "main.py"]
}
]
},
"test": {
"engine": "docker",
"entry": "test_main.py",
"steps": [
{
"box": "python",
"command": ["python", "-m", "unittest"],
"noutput": 8192
}
]
}
}
}

View File

@ -1,12 +0,0 @@
{
"run": {
"engine": "docker",
"entry": "main.sh",
"steps": [
{
"box": "alpine",
"command": ["sh", "main.sh"]
}
]
}
}

View File

@ -1,125 +0,0 @@
# Adding a sandbox
A _sandbox_ is an isolated execution environment for running code snippets. A sandbox is typically implemented as one or more Docker containers. A sandbox supports at least one _command_ (usually the `run` one), but it can support more (like `test` or any other).
Codapi comes with a single `sh` sandbox preinstalled, but you can easily add others. Let's see some examples.
## Python
First, let's create a Docker image capable of running Python with some third-party packages:
```sh
cd /opt/codapi
mkdir images/python
touch images/python/Dockerfile
touch images/python/requirements.txt
```
Fill the `Dockerfile`:
```Dockerfile
FROM python:3.11-alpine
RUN adduser --home /sandbox --disabled-password sandbox
COPY requirements.txt /tmp
RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt
USER sandbox
WORKDIR /sandbox
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
```
And the `requirements.txt`:
```
numpy==1.26.2
pandas==2.1.3
```
Build the image:
```sh
docker build --file images/python/Dockerfile --tag codapi/python:latest images/python/
```
And register the image as a Codapi _box_ in `configs/boxes.json`:
```js
{
// ...
"python": {
"image": "codapi/python"
}
}
```
Finally, let's configure what happens when the client executes the `run` command in the `python` sandbox. To do this, we create `configs/commands/python.json`:
```js
{
"run": {
"engine": "docker",
"entry": "main.py",
"steps": [
{
"box": "python",
"command": ["python", "main.py"]
}
]
}
}
```
This is essentially what it says:
> When the client executes the `run` command in the `python` sandbox, save their code to the `main.py` file, then run it in the `python` box (Docker container) using the `python main.py` shell command.
What if we want to add another command (say, `test`) to the same sandbox? Let's edit `configs/commands/python.json` again:
```js
{
"run": {
// ...
},
"test": {
"engine": "docker",
"entry": "test_main.py",
"steps": [
{
"box": "python",
"command": ["python", "-m", "unittest"],
"noutput": 8192
}
]
}
}
```
Besides configuring a different shell command, here we increased the maximum output size to 8Kb, as tests tend to be quite chatty (you can see the default value in `configs/config.json`).
To apply the changed configuration, restart Codapi (as root):
```sh
systemctl restart codapi.service
```
And try running some Python code:
```sh
curl -H "content-type: application/json" -d '{ "sandbox": "python", "command": "run", "files": {"": "print(42)" }}' http://localhost:1313/v1/exec
```
Which produces the following output:
```json
{
"id": "python_run_7683de5a",
"ok": true,
"duration": 252,
"stdout": "42\n",
"stderr": ""
}
```

View File

@ -1,42 +0,0 @@
# API
Call `/v1/exec` to run the code in a sandbox:
```http
POST https://api.codapi.org/v1/exec
content-type: application/json
{
"sandbox": "python",
"command": "run",
"files": {
"": "print('hello world')"
}
}
```
`sandbox` is the name of the pre-configured sandbox, and `command` is the name of a command supported by that sandbox. See [Adding a sandbox](add-sandbox.md) for details on how to add a new sandbox.
`files` is a map, where the key is a filename and the value is its contents. When executing a single file, it should either be named as the `command` expects, or be an empty string (as in the example above).
Response:
```http
HTTP/1.1 200 OK
Content-Type: application/json
{
"id": "python_run_9b7b1afd",
"ok": true,
"duration": 314,
"stdout": "hello world\n",
"stderr": ""
}
```
- `id` is the unique execution identifier.
- `ok` is `true` if the code executed without errors, or `false` otherwise.
- `duration` is the execution time in milliseconds.
- `stdout` is what the code printed to the standard output.
- `stderr` is what the code printed to the standard error, or a compiler/os error (if any).
-

View File

@ -1,7 +1,5 @@
# Installing Codapi
Make sure you install Codapi on a separate machine — this is a must for security reasons. Do not store any sensitive data or credentials on this machine. This way, even if someone runs malicious code that somehow escapes the isolated environment, they won't have access to your other machines and data.
Steps for Debian (11/12) or Ubuntu (20.04/22.04).
1. Install necessary packages (as root):
@ -28,24 +26,24 @@ docker run hello-world
```sh
cd /opt/codapi
curl -L -O "https://github.com/nalgeon/codapi/releases/download/0.8.0/codapi_0.8.0_linux_amd64.tar.gz"
tar xvzf codapi_0.8.0_linux_amd64.tar.gz
chmod +x codapi
rm -f codapi_0.8.0_linux_amd64.tar.gz
curl -L -o codapi.zip "https://api.github.com/repos/nalgeon/codapi/actions/artifacts/926428361/zip"
unzip -u codapi.zip
chmod +x build/codapi
rm -f codapi.zip
```
5. Build Docker images (as codapi):
6. Build Docker images (as codapi):
```sh
cd /opt/codapi
docker build --file images/alpine/Dockerfile --tag codapi/alpine:latest images/alpine/
make images
```
6. Verify that Codapi starts without errors (as codapi):
7. Verify that Codapi starts without errors (as codapi):
```sh
cd /opt/codapi
./codapi
./build/codapi
```
Should print the `alpine` box and the `sh` command:
@ -60,7 +58,7 @@ Should print the `alpine` box and the `sh` command:
Stop it with Ctrl+C.
7. Configure Codapi as systemd service (as root):
8. Configure Codapi as systemd service (as root):
```sh
mv /opt/codapi/codapi.service /etc/systemd/system/
@ -84,7 +82,7 @@ codapi.service - Code playgrounds
...
```
8. Verify that Codapi is working:
9. Verify that Codapi is working:
```sh
curl -H "content-type: application/json" -d '{ "sandbox": "sh", "command": "run", "files": {"": "echo hello" }}' http://localhost:1313/v1/exec

View File

@ -8,14 +8,15 @@ import (
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/internal/fileio"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/execy"
"github.com/nalgeon/codapi/fileio"
"github.com/nalgeon/codapi/logx"
)
var killTimeout = 5 * time.Second
@ -23,7 +24,6 @@ var killTimeout = 5 * time.Second
const (
actionRun = "run"
actionExec = "exec"
actionStop = "stop"
)
// A Docker engine executes a specific sandbox command
@ -42,7 +42,7 @@ func NewDocker(cfg *config.Config, sandbox, command string) Engine {
// Exec executes the command and returns the output.
func (e *Docker) Exec(req Request) Execution {
// all steps operate in the same temp directory
dir, err := fileio.MkdirTemp(0777)
dir, err := os.MkdirTemp("", "")
if err != nil {
err = NewExecutionError("create temp dir", err)
return Fail(req.ID, err)
@ -54,10 +54,7 @@ func (e *Docker) Exec(req Request) Execution {
if e.cmd.Entry != "" {
// write request files to the temp directory
err = e.writeFiles(dir, req.Files)
var argErr ArgumentError
if errors.As(err, &argErr) {
return Fail(req.ID, err)
} else if err != nil {
if err != nil {
err = NewExecutionError("write files to temp dir", err)
return Fail(req.ID, err)
}
@ -65,7 +62,7 @@ func (e *Docker) Exec(req Request) Execution {
// initialization step
if e.cmd.Before != nil {
out := e.execStep(e.cmd.Before, req, dir, nil)
out := e.execStep(e.cmd.Before, req.ID, dir, nil)
if !out.OK {
return out
}
@ -73,14 +70,14 @@ func (e *Docker) Exec(req Request) Execution {
// the first step is required
first, rest := e.cmd.Steps[0], e.cmd.Steps[1:]
out := e.execStep(first, req, dir, req.Files)
out := e.execStep(first, req.ID, dir, req.Files)
// the rest are optional
if out.OK && len(rest) > 0 {
// each step operates on the results of the previous one,
// without using the source files - hence `nil` instead of `files`
for _, step := range rest {
out = e.execStep(step, req, dir, nil)
out = e.execStep(step, req.ID, dir, nil)
if !out.OK {
break
}
@ -89,7 +86,7 @@ func (e *Docker) Exec(req Request) Execution {
// cleanup step
if e.cmd.After != nil {
afterOut := e.execStep(e.cmd.After, req, dir, nil)
afterOut := e.execStep(e.cmd.After, req.ID, dir, nil)
if out.OK && !afterOut.OK {
return afterOut
}
@ -99,67 +96,34 @@ func (e *Docker) Exec(req Request) Execution {
}
// execStep executes a step using the docker container.
func (e *Docker) execStep(step *config.Step, req Request, dir string, files Files) Execution {
box, err := e.getBox(step, req)
if err != nil {
return Fail(req.ID, err)
}
err = e.copyFiles(box, dir)
func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution {
box := e.cfg.Boxes[step.Box]
err := e.copyFiles(box, dir)
if err != nil {
err = NewExecutionError("copy files to temp dir", err)
return Fail(req.ID, err)
return Fail(reqID, err)
}
stdout, stderr, err := e.exec(box, step, req, dir, files)
stdout, stderr, err := e.exec(box, step, reqID, dir, files)
if err != nil {
return Fail(req.ID, err)
return Fail(reqID, err)
}
return Execution{
ID: req.ID,
ID: reqID,
OK: true,
Stdout: stdout,
Stderr: stderr,
}
}
// getBox selects an appropriate box for the step (if any).
func (e *Docker) getBox(step *config.Step, req Request) (*config.Box, error) {
if step.Action != actionRun {
// steps other than "run" use existing containers
// and do not spin up new ones
return nil, nil
}
var boxName string
// If the version is set in the step config, use it.
if step.Version != "" {
if step.Version == "latest" {
boxName = step.Box
} else {
boxName = step.Box + ":" + step.Version
}
} else if req.Version != "" {
// If the version is set in the request, use it.
boxName = step.Box + ":" + req.Version
} else {
// otherwise, use the latest version
boxName = step.Box
}
box, found := e.cfg.Boxes[boxName]
if !found {
return nil, fmt.Errorf("unknown box %s", boxName)
}
return box, nil
}
// copyFiles copies box files to the temporary directory.
func (e *Docker) copyFiles(box *config.Box, dir string) error {
if box == nil || len(box.Files) == 0 {
return nil
}
for _, pattern := range box.Files {
err := fileio.CopyFiles(pattern, dir, 0444)
err := fileio.CopyFiles(pattern, dir)
if err != nil {
return err
}
@ -174,13 +138,8 @@ func (e *Docker) writeFiles(dir string, files Files) error {
if name == "" {
name = e.cmd.Entry
}
var path string
path, err = fileio.JoinDir(dir, name)
if err != nil {
err = NewArgumentError(fmt.Sprintf("files[%s]", name), err)
return false
}
err = fileio.WriteFile(path, content, 0444)
path := filepath.Join(dir, name)
err = os.WriteFile(path, []byte(content), 0444)
return err == nil
})
return err
@ -188,18 +147,18 @@ func (e *Docker) writeFiles(dir string, files Files) error {
// exec executes the step in the docker container
// using the files from in the temporary directory.
func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir string, files Files) (stdout string, stderr string, err error) {
func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) {
// limit the stdout/stderr size
prog := NewProgram(step.Timeout, int64(step.NOutput))
args := e.buildArgs(box, step, req, dir)
args := e.buildArgs(box, step, reqID, dir)
if step.Stdin {
// pass files to container from stdin
stdin := filesReader(files)
stdout, stderr, err = prog.RunStdin(stdin, req.ID, "docker", args...)
stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...)
} else {
// pass files to container from temp directory
stdout, stderr, err = prog.Run(req.ID, "docker", args...)
stdout, stderr, err = prog.Run(reqID, "docker", args...)
}
if err == nil {
@ -209,15 +168,15 @@ func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir strin
if err.Error() == "signal: killed" {
if step.Action == actionRun {
// we have to "docker kill" the container here, because the process
// we have to "docker kill" the container here, because the proccess
// inside the container is not related to the "docker run" process,
// and will hang forever after the "docker run" process is killed
go func() {
err = dockerKill(req.ID)
err = dockerKill(reqID)
if err == nil {
logx.Debug("%s: docker kill ok", req.ID)
logx.Debug("%s: docker kill ok", reqID)
} else {
logx.Log("%s: docker kill failed: %v", req.ID, err)
logx.Log("%s: docker kill failed: %v", reqID, err)
}
}()
}
@ -243,31 +202,28 @@ func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir strin
}
// buildArgs prepares the arguments for the `docker` command.
func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string {
var args []string
switch step.Action {
case actionRun:
args = dockerRunArgs(box, step, req, dir)
case actionExec:
args = dockerExecArgs(step, req)
case actionStop:
args = dockerStopArgs(step, req)
default:
if step.Action == actionRun {
args = dockerRunArgs(box, step, name, dir)
} else if step.Action == actionExec {
args = dockerExecArgs(step)
} else {
// should never happen if the config is valid
args = []string{"version"}
}
command := expandVars(step.Command, req.ID)
command := expandVars(step.Command, name)
args = append(args, command...)
logx.Debug("%v", args)
return args
}
// buildArgs prepares the arguments for the `docker run` command.
func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string {
args := []string{
actionRun, "--rm",
"--name", req.ID,
"--name", name,
"--runtime", box.Runtime,
"--cpus", strconv.Itoa(box.CPU),
"--memory", fmt.Sprintf("%dm", box.Memory),
@ -275,15 +231,12 @@ func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string)
"--pids-limit", strconv.Itoa(box.NProc),
"--user", step.User,
}
if step.Detach {
args = append(args, "--detach")
if !box.Writable {
args = append(args, "--read-only")
}
if step.Stdin {
args = append(args, "--interactive")
}
if !box.Writable {
args = append(args, "--read-only")
}
if box.Storage != "" {
args = append(args, "--storage-opt", fmt.Sprintf("size=%s", box.Storage))
}
@ -307,23 +260,14 @@ func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string)
}
// dockerExecArgs prepares the arguments for the `docker exec` command.
func dockerExecArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
func dockerExecArgs(step *config.Step) []string {
return []string{
actionExec, "--interactive",
"--user", step.User,
box,
step.Box,
}
}
// dockerStopArgs prepares the arguments for the `docker stop` command.
func dockerStopArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
return []string{actionStop, box}
}
// filesReader creates a reader over an in-memory collection of files.
func filesReader(files Files) io.Reader {
var input strings.Builder
@ -339,7 +283,7 @@ func expandVars(command []string, name string) []string {
expanded := make([]string, len(command))
copy(expanded, command)
for i, cmd := range expanded {
expanded[i] = strings.Replace(cmd, ":name", name, -1)
expanded[i] = strings.Replace(cmd, ":name", name, 1)
}
return expanded
}

164
engine/docker_test.go Normal file
View File

@ -0,0 +1,164 @@
package engine
import (
"strings"
"testing"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/execy"
"github.com/nalgeon/codapi/logx"
)
var dockerCfg = &config.Config{
Boxes: map[string]*config.Box{
"postgresql": {
Image: "codapi/postgresql",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"python": {
Image: "codapi/python",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
},
Commands: map[string]config.SandboxCommands{
"postgresql": map[string]*config.Command{
"run": {
Engine: "docker",
Before: &config.Step{
Box: "postgres", User: "sandbox", Action: "exec",
Command: []string{"psql", "-f", "create.sql"},
NOutput: 4096,
},
Steps: []*config.Step{
{
Box: "postgres", User: "sandbox", Action: "exec", Stdin: true,
Command: []string{"psql", "--user=:name"},
NOutput: 4096,
},
},
After: &config.Step{
Box: "postgres", User: "sandbox", Action: "exec",
Command: []string{"psql", "-f", "drop.sql"},
NOutput: 4096,
},
},
},
"python": map[string]*config.Command{
"run": {
Engine: "docker",
Entry: "main.py",
Steps: []*config.Step{
{
Box: "python", User: "sandbox", Action: "run",
Command: []string{"python", "main.py"},
NOutput: 4096,
},
},
},
},
},
}
func TestDockerRun(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker run": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "python", "run")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello world"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "python main.py")
})
}
func TestDockerExec(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker exec": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "postgresql", "run")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "http_42",
Sandbox: "postgresql",
Command: "run",
Files: map[string]string{
"": "select 'hello world'",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello world"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "psql -f create.sql")
mem.MustHave(t, "psql --user=http_42")
mem.MustHave(t, "psql -f drop.sql")
})
}
func Test_expandVars(t *testing.T) {
const name = "codapi_01"
commands := map[string]string{
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
}
for cmd, want := range commands {
src := strings.Fields(cmd)
exp := expandVars(src, name)
got := strings.Join(exp, " ")
if got != want {
t.Errorf("%q: expected %q, got %q", cmd, got, want)
}
}
}

View File

@ -5,25 +5,20 @@ import (
"errors"
"fmt"
"github.com/nalgeon/codapi/internal/stringx"
"github.com/nalgeon/codapi/stringx"
)
// A Request initiates code execution.
type Request struct {
ID string `json:"id"`
Sandbox string `json:"sandbox"`
Version string `json:"version,omitempty"`
Command string `json:"command"`
Files Files `json:"files"`
}
// GenerateID() sets a unique ID for the request.
func (r *Request) GenerateID() {
if r.Version != "" {
r.ID = fmt.Sprintf("%s.%s_%s_%s", r.Sandbox, r.Version, r.Command, stringx.RandString(8))
} else {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
}
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
}
// An Execution is an output from the code execution engine.
@ -62,25 +57,6 @@ func (err ExecutionError) Unwrap() error {
return err.inner
}
// An ArgumentError is returned if code execution failed
// due to the invalid value of the request argument.
type ArgumentError struct {
name string
reason error
}
func NewArgumentError(name string, reason error) ArgumentError {
return ArgumentError{name: name, reason: reason}
}
func (err ArgumentError) Error() string {
return err.name + ": " + err.reason.Error()
}
func (err ArgumentError) Unwrap() error {
return err.reason
}
// Files are a collection of files to be executed by the engine.
type Files map[string]string

View File

@ -4,34 +4,9 @@ import (
"errors"
"reflect"
"sort"
"strings"
"testing"
)
func TestGenerateID(t *testing.T) {
t.Run("with version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Version: "dev",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python.dev_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
t.Run("without version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
}
func TestExecutionError(t *testing.T) {
inner := errors.New("inner error")
err := NewExecutionError("failed", inner)

View File

@ -7,8 +7,8 @@ import (
"strings"
"time"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/execy"
"github.com/nalgeon/codapi/logx"
)
// A Program is an executable program.
@ -48,7 +48,7 @@ func (p *Program) RunStdin(stdin io.Reader, id, name string, arg ...string) (std
cmd.Stdout = LimitWriter(&cmdout, p.nOutput)
cmd.Stderr = LimitWriter(&cmderr, p.nOutput)
err = execy.Run(cmd)
stdout = strings.TrimSpace(cmdout.String())
stderr = strings.TrimSpace(cmderr.String())
stdout = cmdout.String()
stderr = cmderr.String()
return
}

View File

@ -5,7 +5,7 @@ import (
"strings"
"testing"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/execy"
)
func TestProgram_Run(t *testing.T) {

View File

@ -9,9 +9,9 @@ import (
"net/http"
"strings"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/httpx"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/httpx"
"github.com/nalgeon/codapi/logx"
)
// An HTTP engine sends HTTP requests.

View File

@ -5,9 +5,9 @@ import (
"net/http"
"testing"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/httpx"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/httpx"
"github.com/nalgeon/codapi/logx"
)
var httpCfg = &config.Config{

View File

@ -4,7 +4,7 @@ import (
"os/exec"
"strings"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/logx"
)
// Mock installs mock outputs for given commands.

39
fileio/fileio.go Normal file
View File

@ -0,0 +1,39 @@
// Package fileio provides high-level file operations.
package fileio
import (
"io"
"os"
"path/filepath"
)
// CopyFile copies all files matching the pattern
// to the destination directory.
func CopyFiles(pattern string, dstDir string) error {
matches, err := filepath.Glob(pattern)
if err != nil {
return err
}
for _, match := range matches {
src, err := os.Open(match)
if err != nil {
return err
}
defer src.Close()
dstFile := filepath.Join(dstDir, filepath.Base(match))
dst, err := os.Create(dstFile)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
return err
}
}
return nil
}

56
fileio/fileio_test.go Normal file
View File

@ -0,0 +1,56 @@
package fileio
import (
"os"
"path/filepath"
"testing"
)
func TestCopyFiles(t *testing.T) {
// Create a temporary directory for testing
srcDir, err := os.MkdirTemp("", "src")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(srcDir)
// Create a source file
srcFile := filepath.Join(srcDir, "source.txt")
err = os.WriteFile(srcFile, []byte("test data"), 0644)
if err != nil {
t.Fatal(err)
}
// Specify the destination directory
dstDir, err := os.MkdirTemp("", "dst")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dstDir)
// Call the CopyFiles function
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir)
if err != nil {
t.Fatal(err)
}
// Verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
_, err = os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
// Read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// Verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
}

2
go.mod
View File

@ -1,3 +1,3 @@
module github.com/nalgeon/codapi
go 1.21
go 1.20

View File

@ -1,109 +0,0 @@
package config
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"github.com/nalgeon/codapi/internal/fileio"
)
const (
configFilename = "config.json"
boxesFilename = "boxes.json"
commandsDirname = "commands"
)
// Read reads application config from JSON files.
func Read(path string) (*Config, error) {
cfg, err := ReadConfig(filepath.Join(path, configFilename))
if err != nil {
return nil, err
}
cfg, err = ReadBoxes(cfg, filepath.Join(path, boxesFilename))
if err != nil {
return nil, err
}
cfg, err = ReadCommands(cfg, filepath.Join(path, commandsDirname))
if err != nil {
return nil, err
}
return cfg, err
}
// ReadConfig reads application config from a JSON file.
func ReadConfig(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
cfg := &Config{}
err = json.Unmarshal(data, cfg)
if err != nil {
return nil, err
}
return cfg, err
}
// ReadBoxes reads boxes config from a JSON file.
func ReadBoxes(cfg *Config, path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
boxes := make(map[string]*Box)
err = json.Unmarshal(data, &boxes)
if err != nil {
return nil, err
}
for _, box := range boxes {
setBoxDefaults(box, cfg.Box)
}
cfg.Boxes = boxes
return cfg, err
}
// ReadCommands reads commands config from a JSON file.
func ReadCommands(cfg *Config, path string) (*Config, error) {
fnames, err := filepath.Glob(filepath.Join(path, "*.json"))
if err != nil {
return nil, err
}
cfg.Commands = make(map[string]SandboxCommands, len(fnames))
for _, fname := range fnames {
sandbox := strings.TrimSuffix(filepath.Base(fname), ".json")
commands, err := fileio.ReadJson[SandboxCommands](fname)
if err != nil {
break
}
setCommandDefaults(commands, cfg)
cfg.Commands[sandbox] = commands
}
return cfg, err
}
// setCommandDefaults applies global defaults to sandbox commands.
func setCommandDefaults(commands SandboxCommands, cfg *Config) {
for _, cmd := range commands {
if cmd.Before != nil {
setStepDefaults(cmd.Before, cfg.Step)
}
for _, step := range cmd.Steps {
setStepDefaults(step, cfg.Step)
}
if cmd.After != nil {
setStepDefaults(cmd.After, cfg.Step)
}
}
}

View File

@ -1,23 +0,0 @@
{
"run": {
"engine": "docker",
"entry": "main.py",
"steps": [
{
"box": "python",
"command": ["python", "main.py"]
}
]
},
"test": {
"engine": "docker",
"entry": "test_main.py",
"steps": [
{
"box": "python",
"command": ["python", "-m", "unittest"],
"noutput": 8192
}
]
}
}

View File

@ -1,379 +0,0 @@
package engine
import (
"fmt"
"strings"
"testing"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/internal/logx"
)
var dockerCfg = &config.Config{
Boxes: map[string]*config.Box{
"alpine": {
Image: "codapi/alpine",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go": {
Image: "codapi/go",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go:dev": {
Image: "codapi/go:dev",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"python": {
Image: "codapi/python",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"python:dev": {
Image: "codapi/python:dev",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
},
Commands: map[string]config.SandboxCommands{
"alpine": map[string]*config.Command{
"echo": {
Engine: "docker",
Before: &config.Step{
Box: "alpine", User: "sandbox", Action: "run", Detach: true,
Command: []string{"echo", "before"},
NOutput: 4096,
},
Steps: []*config.Step{
{
Box: ":name", User: "sandbox", Action: "exec",
Command: []string{"sh", "main.sh"},
NOutput: 4096,
},
},
After: &config.Step{
Box: ":name", User: "sandbox", Action: "stop",
NOutput: 4096,
},
},
},
"go": map[string]*config.Command{
"run": {
Engine: "docker",
Steps: []*config.Step{
{
Box: "go", User: "sandbox", Action: "run",
Command: []string{"go", "build"},
NOutput: 4096,
},
{
Box: "alpine", Version: "latest",
User: "sandbox", Action: "run",
Command: []string{"./main"},
NOutput: 4096,
},
},
},
},
"postgresql": map[string]*config.Command{
"run": {
Engine: "docker",
Before: &config.Step{
Box: "postgres", User: "sandbox", Action: "exec",
Command: []string{"psql", "-f", "create.sql"},
NOutput: 4096,
},
Steps: []*config.Step{
{
Box: "postgres", User: "sandbox", Action: "exec", Stdin: true,
Command: []string{"psql", "--user=:name"},
NOutput: 4096,
},
},
After: &config.Step{
Box: "postgres", User: "sandbox", Action: "exec",
Command: []string{"psql", "-f", "drop.sql"},
NOutput: 4096,
},
},
},
"python": map[string]*config.Command{
"run": {
Engine: "docker",
Entry: "main.py",
Steps: []*config.Step{
{
Box: "python", User: "sandbox", Action: "run",
Command: []string{"python", "main.py"},
NOutput: 4096,
},
},
},
},
},
}
func TestDockerRun(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker run": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
t.Run("success", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello world"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "codapi/python")
mem.MustHave(t, "python main.py")
})
t.Run("latest version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python")
})
t.Run("custom version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python:dev")
})
t.Run("step version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "go", "run")
req := Request{
ID: "http_42",
Sandbox: "go",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "var n = 42",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/go:dev")
mem.MustHave(t, "codapi/alpine")
})
t.Run("unsupported version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "42",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := "unknown box python:42"
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
t.Run("directory traversal attack", func(t *testing.T) {
mem.Clear()
const fileName = "../../opt/codapi/codapi"
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
fileName: "hehe",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := fmt.Sprintf("files[%s]: invalid name", fileName)
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
}
func TestDockerExec(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker exec": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "postgresql", "run")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "http_42",
Sandbox: "postgresql",
Command: "run",
Files: map[string]string{
"": "select 'hello world'",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello world"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "psql -f create.sql")
mem.MustHave(t, "psql --user=http_42")
mem.MustHave(t, "psql -f drop.sql")
})
}
func TestDockerStop(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker run": {Stdout: "c958ff2", Stderr: "", Err: nil},
"docker exec": {Stdout: "hello", Stderr: "", Err: nil},
"docker stop": {Stdout: "alpine_42", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "alpine", "echo")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "alpine_42",
Sandbox: "alpine",
Command: "echo",
Files: map[string]string{
"": "echo hello",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "docker run --rm --name alpine_42", "--detach")
mem.MustHave(t, "docker exec --interactive --user sandbox alpine_42 sh main.sh")
mem.MustHave(t, "docker stop alpine_42")
})
}
func Test_expandVars(t *testing.T) {
const name = "codapi_01"
commands := map[string]string{
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
"sh copy.sh :name new-:name": "sh copy.sh " + name + " new-" + name,
}
for cmd, want := range commands {
src := strings.Fields(cmd)
exp := expandVars(src, name)
got := strings.Join(exp, " ")
if got != want {
t.Errorf("%q: expected %q, got %q", cmd, got, want)
}
}
}

View File

@ -1,146 +0,0 @@
// Package fileio provides high-level file operations.
package fileio
import (
"encoding/base64"
"encoding/json"
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
)
// Exists checks if the specified path exists.
func Exists(path string) bool {
_, err := os.Stat(path)
// we need a double negation here, because
// errors.Is(err, os.ErrExist)
// does not work
return !errors.Is(err, os.ErrNotExist)
}
// CopyFile copies all files matching the pattern
// to the destination directory. Does not overwrite existing file.
func CopyFiles(pattern string, dstDir string, perm fs.FileMode) error {
matches, err := filepath.Glob(pattern)
if err != nil {
return err
}
for _, match := range matches {
src, err := os.Open(match)
if err != nil {
return err
}
defer src.Close()
dstFile := filepath.Join(dstDir, filepath.Base(match))
if Exists(dstFile) {
continue
}
dst, err := os.OpenFile(dstFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
return err
}
}
return nil
}
// ReadJson reads the file and decodes it from JSON.
func ReadJson[T any](path string) (T, error) {
var obj T
data, err := os.ReadFile(path)
if err != nil {
return obj, err
}
err = json.Unmarshal(data, &obj)
if err != nil {
return obj, err
}
return obj, err
}
// WriteFile writes the file to disk.
// The content can be text or binary (encoded as a data URL),
// e.g. data:application/octet-stream;base64,MTIz
func WriteFile(path, content string, perm fs.FileMode) (err error) {
var data []byte
if !strings.HasPrefix(content, "data:") {
// text file
data = []byte(content)
return os.WriteFile(path, data, perm)
}
// data-url encoded file
meta, encoded, found := strings.Cut(content, ",")
if !found {
return errors.New("invalid data-url encoding")
}
if !strings.HasSuffix(meta, "base64") {
// no need to decode
data = []byte(encoded)
return os.WriteFile(path, data, perm)
}
// decode base64-encoded data
data, err = base64.StdEncoding.DecodeString(encoded)
if err != nil {
return err
}
return os.WriteFile(path, data, perm)
}
// JoinDir joins a directory path with a relative file path,
// making sure that the resulting path is still inside the directory.
// Returns an error otherwise.
func JoinDir(dir string, name string) (string, error) {
if dir == "" {
return "", errors.New("invalid dir")
}
cleanName := filepath.Clean(name)
if cleanName == "" {
return "", errors.New("invalid name")
}
if cleanName == "." || cleanName == "/" || filepath.IsAbs(cleanName) {
return "", errors.New("invalid name")
}
path := filepath.Join(dir, cleanName)
dirPrefix := filepath.Clean(dir)
if dirPrefix != "/" {
dirPrefix += string(os.PathSeparator)
}
if !strings.HasPrefix(path, dirPrefix) {
return "", errors.New("invalid name")
}
return path, nil
}
// MkdirTemp creates a new temporary directory with given permissions
// and returns the pathname of the new directory.
func MkdirTemp(perm fs.FileMode) (string, error) {
dir, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
err = os.Chmod(dir, perm)
if err != nil {
os.Remove(dir)
return "", err
}
return dir, nil
}

View File

@ -1,385 +0,0 @@
package fileio
import (
"io/fs"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestExists(t *testing.T) {
t.Run("exists", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
err := os.WriteFile(path, []byte{1, 2, 3}, 0444)
if err != nil {
t.Fatal(err)
}
if !Exists(path) {
t.Fatalf("Exists: %s does not exist", filepath.Base(path))
}
})
t.Run("does not exist", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
if Exists(path) {
t.Fatalf("Exists: %s should not exist", filepath.Base(path))
}
})
}
func TestCopyFiles(t *testing.T) {
// create a temporary directory for testing
srcDir, err := os.MkdirTemp("", "src")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(srcDir)
// create a source file
srcFile := filepath.Join(srcDir, "source.txt")
err = os.WriteFile(srcFile, []byte("test data"), 0644)
if err != nil {
t.Fatal(err)
}
// specify the destination directory
dstDir, err := os.MkdirTemp("", "dst")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dstDir)
t.Run("copy", func(t *testing.T) {
// call the CopyFiles function
const perm = fs.FileMode(0444)
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, perm)
if err != nil {
t.Fatal(err)
}
// verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
fileInfo, err := os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
if fileInfo.Mode() != perm {
t.Errorf("unexpected file permissions: got %v, want %v", fileInfo.Mode(), perm)
}
// read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
})
t.Run("skip existing", func(t *testing.T) {
// existing file in the destination dir
path := filepath.Join(dstDir, "existing.txt")
err := os.WriteFile(path, []byte("v1"), 0444)
if err != nil {
t.Fatal(err)
}
// same file in the source dir
path = filepath.Join(srcDir, "existing.txt")
err = os.WriteFile(path, []byte("v2"), 0444)
if err != nil {
t.Fatal(err)
}
// copy files
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, 0444)
if err != nil {
t.Fatal(err)
}
// verify that the new file was copied correctly
newFile := filepath.Join(dstDir, "source.txt")
_, err = os.Stat(newFile)
if err != nil {
t.Fatalf("new file not copied: %s", err)
}
// verify that the existing file remained unchanged
existFile := filepath.Join(dstDir, "existing.txt")
data, err := os.ReadFile(existFile)
if err != nil {
t.Fatal(err)
}
expected := []byte("v1")
if string(data) != string(expected) {
t.Error("existing file got overwritten")
}
})
}
func TestReadJson(t *testing.T) {
type Person struct{ Name string }
t.Run("valid", func(t *testing.T) {
got, err := ReadJson[Person](filepath.Join("testdata", "valid.json"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
want := Person{"alice"}
if !reflect.DeepEqual(got, want) {
t.Errorf("expected %v, got %v", want, got)
}
})
t.Run("invalid", func(t *testing.T) {
_, err := ReadJson[Person](filepath.Join("testdata", "invalid.json"))
if err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("does not exist", func(t *testing.T) {
_, err := ReadJson[Person](filepath.Join("testdata", "missing.json"))
if err == nil {
t.Fatal("expected error, got nil")
}
})
}
func TestWriteFile(t *testing.T) {
dir, err := os.MkdirTemp("", "files")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
t.Run("text", func(t *testing.T) {
path := filepath.Join(dir, "data.txt")
err = WriteFile(path, "hello", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("hello")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-octet-stream", func(t *testing.T) {
path := filepath.Join(dir, "data-1.bin")
err = WriteFile(path, "data:application/octet-stream;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-base64", func(t *testing.T) {
path := filepath.Join(dir, "data-2.bin")
err = WriteFile(path, "data:;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-text-plain", func(t *testing.T) {
path := filepath.Join(dir, "data-3.bin")
err = WriteFile(path, "data:text/plain;,123", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("perm", func(t *testing.T) {
const perm = 0444
path := filepath.Join(dir, "perm.txt")
err = WriteFile(path, "hello", perm)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
fileInfo, err := os.Stat(path)
if err != nil {
t.Fatalf("file not created: %s", err)
}
if fileInfo.Mode().Perm() != perm {
t.Errorf("unexpected file permissions: expected %o, got %o", perm, fileInfo.Mode().Perm())
}
})
t.Run("missing data-url separator", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
err = WriteFile(path, "data:application/octet-stream:MTIz", 0444)
if err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("invalid binary value", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
err = WriteFile(path, "data:;base64,12345", 0444)
if err == nil {
t.Fatal("expected error, got nil")
}
})
}
func TestJoinDir(t *testing.T) {
tests := []struct {
name string
dir string
filename string
want string
wantErr bool
}{
{
name: "regular join",
dir: "/home/user",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "join with dot",
dir: "/home/user",
filename: ".",
want: "",
wantErr: true,
},
{
name: "join with absolute path",
dir: "/home/user",
filename: "/etc/passwd",
want: "",
wantErr: true,
},
{
name: "join with parent directory",
dir: "/home/user",
filename: "../user2/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "empty directory",
dir: "",
filename: "report.txt",
want: "",
wantErr: true,
},
{
name: "empty filename",
dir: "/home/user",
filename: "",
want: "",
wantErr: true,
},
{
name: "directory with trailing slash",
dir: "/home/user/",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "filename with leading slash",
dir: "/home/user",
filename: "/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "root directory",
dir: "/",
filename: "report.txt",
want: "/report.txt",
wantErr: false,
},
{
name: "dot dot slash filename",
dir: "/home/user",
filename: "..",
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := JoinDir(tt.dir, tt.filename)
if (err != nil) != tt.wantErr {
t.Errorf("JoinDir() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("JoinDir() = %v, want %v", got, tt.want)
}
})
}
}
func TestMkdirTemp(t *testing.T) {
t.Run("default permissions", func(t *testing.T) {
const perm = 0755
dir, err := MkdirTemp(perm)
if err != nil {
t.Fatalf("failed to create temp directory: %v", err)
}
defer os.Remove(dir)
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("failed to stat temp directory: %v", err)
}
if info.Mode().Perm() != perm {
t.Errorf("unexpected permissions: expected %o, got %o", perm, info.Mode().Perm())
}
})
t.Run("non-default permissions", func(t *testing.T) {
const perm = 0777
dir, err := MkdirTemp(perm)
if err != nil {
t.Fatalf("failed to create temp directory: %v", err)
}
defer os.Remove(dir)
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("failed to stat temp directory: %v", err)
}
if info.Mode().Perm() != perm {
t.Errorf("unexpected permissions: expected %o, got %o", perm, info.Mode().Perm())
}
})
}

View File

@ -1 +0,0 @@
name: alice

View File

@ -1,3 +0,0 @@
{
"name": "alice"
}

View File

@ -30,16 +30,9 @@ func (m *Memory) WriteString(s string) {
}
// Has returns true if the memory has the message.
func (m *Memory) Has(message ...string) bool {
func (m *Memory) Has(msg string) bool {
for _, line := range m.Lines {
containsAll := true
for _, part := range message {
if !strings.Contains(line, part) {
containsAll = false
break
}
}
if containsAll {
if strings.Contains(line, msg) {
return true
}
}
@ -47,29 +40,22 @@ func (m *Memory) Has(message ...string) bool {
}
// MustHave checks if the memory has the message.
// If the message consists of several parts,
// they must all be in the same memory line.
func (m *Memory) MustHave(t *testing.T, message ...string) {
if !m.Has(message...) {
t.Errorf("%s must have: %v", m.Name, message)
func (m *Memory) MustHave(t *testing.T, msg string) {
if !m.Has(msg) {
t.Errorf("%s must have: %s", m.Name, msg)
}
}
// MustNotHave checks if the memory does not have the message.
func (m *Memory) MustNotHave(t *testing.T, message ...string) {
if m.Has(message...) {
t.Errorf("%s must NOT have: %v", m.Name, message)
func (m *Memory) MustNotHave(t *testing.T, msg string) {
if m.Has(msg) {
t.Errorf("%s must NOT have: %s", m.Name, msg)
}
}
// Clear clears the memory.
func (m *Memory) Clear() {
m.Lines = []string{}
}
// Print prints memory lines to stdout.
func (m *Memory) Print() {
for _, line := range m.Lines {
fmt.Println(line)
fmt.Print(line)
}
}

View File

@ -40,23 +40,4 @@ func TestMemory_Has(t *testing.T) {
if !mem.Has("hello world") {
t.Error("Has: unexpected false")
}
_, _ = mem.Write([]byte("one two three four"))
if !mem.Has("one two") {
t.Error("Has: one two: unexpected false")
}
if !mem.Has("two three") {
t.Error("Has: two three: unexpected false")
}
if mem.Has("one three") {
t.Error("Has: one three: unexpected true")
}
if !mem.Has("one", "three") {
t.Error("Has: [one three]: unexpected false")
}
if !mem.Has("one", "three", "four") {
t.Error("Has: [one three four]: unexpected false")
}
if !mem.Has("four", "three") {
t.Error("Has: [four three]: unexpected false")
}
}

View File

@ -4,8 +4,8 @@ package sandbox
import (
"fmt"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/engine"
)
// A semaphore represents available concurrent workers
@ -20,7 +20,7 @@ var engineConstr = map[string]func(*config.Config, string, string) engine.Engine
}
// engines is the registry of command executors.
// Each engine executes a specific command in a specific sandbox.
// Each engine executes a specific command in a specifix sandbox.
// sandbox : command : engine
// TODO: Maybe it's better to create a single instance of each engine
// and pass the sandbox and command as arguments to the Exec.

View File

@ -3,8 +3,8 @@ package sandbox
import (
"testing"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/engine"
)
var cfg = &config.Config{

View File

@ -7,7 +7,7 @@ import (
"strings"
"time"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/engine"
)
var ErrUnknownSandbox = errors.New("unknown sandbox")

View File

@ -4,8 +4,8 @@ import (
"errors"
"testing"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/engine"
"github.com/nalgeon/codapi/execy"
)
func TestValidate(t *testing.T) {

View File

@ -8,7 +8,7 @@ import (
"testing"
"time"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/engine"
)
func Test_readJson(t *testing.T) {

View File

@ -7,7 +7,7 @@ import "net/http"
func enableCORS(handler func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("access-control-allow-origin", "*")
w.Header().Set("access-control-allow-methods", "options, post")
w.Header().Set("access-control-allow-method", "post")
w.Header().Set("access-control-allow-headers", "authorization, content-type")
w.Header().Set("access-control-max-age", "3600")
if r.Method == http.MethodOptions {

View File

@ -31,8 +31,8 @@ func Test_enableCORS(t *testing.T) {
if w.Header().Get("access-control-allow-origin") != "*" {
t.Errorf("invalid access-control-allow-origin")
}
if w.Header().Get("access-control-allow-methods") != "options, post" {
t.Errorf("invalid access-control-allow-methods")
if w.Header().Get("access-control-allow-method") != "post" {
t.Errorf("invalid access-control-allow-method")
}
if w.Header().Get("access-control-allow-headers") != "authorization, content-type" {
t.Errorf("invalid access-control-allow-headers")

View File

@ -6,10 +6,10 @@ import (
"fmt"
"net/http"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/internal/sandbox"
"github.com/nalgeon/codapi/internal/stringx"
"github.com/nalgeon/codapi/engine"
"github.com/nalgeon/codapi/logx"
"github.com/nalgeon/codapi/sandbox"
"github.com/nalgeon/codapi/stringx"
)
// NewRouter creates HTTP routes and handlers for them.

View File

@ -7,10 +7,10 @@ import (
"net/http/httptest"
"testing"
"github.com/nalgeon/codapi/internal/config"
"github.com/nalgeon/codapi/internal/engine"
"github.com/nalgeon/codapi/internal/execy"
"github.com/nalgeon/codapi/internal/sandbox"
"github.com/nalgeon/codapi/config"
"github.com/nalgeon/codapi/engine"
"github.com/nalgeon/codapi/execy"
"github.com/nalgeon/codapi/sandbox"
)
var cfg = &config.Config{

View File

@ -8,7 +8,7 @@ import (
"sync"
"time"
"github.com/nalgeon/codapi/internal/logx"
"github.com/nalgeon/codapi/logx"
)
// The maximum duration of the server graceful shutdown.