Benchmarks

#!/usr/bin/env bash
#
# The workload. ONE writer, used by every arm of the comparison, so the only thing that differs
# between arms is what sits between this script and the bucket.
#
#   writer.sh <shape> <target-dir> <files> <file-bytes>
#
# It prints one line to stdout when it is done:
#
#   WRITER wall_ms=<n> bytes=<n> files=<n>
#
# Nothing here is clever on purpose. dd with a fixed block size, no parallelism, no fsync games:
# a benchmark whose writer is doing something subtle measures the writer.
set -euo pipefail

SHAPE="${1:?shape}"
TARGET="${2:?target directory}"
FILES="${3:?file count}"
FILE_BYTES="${4:?bytes per file}"

BLOCK=65536
COUNT=$(( FILE_BYTES / BLOCK ))
[[ "$COUNT" -lt 1 ]] && COUNT=1

mkdir -p "$TARGET"

start_ns() { date +%s%N; }
START="$(start_ns)"
WRITTEN=0

case "$SHAPE" in
  many-small|one-large)
    for i in $(seq 1 "$FILES"); do
      dd if=/dev/zero of="$TARGET/payload-$i.bin" bs="$BLOCK" count="$COUNT" \
         conv=notrunc status=none
      WRITTEN=$(( WRITTEN + COUNT * BLOCK ))
    done
    ;;

  append-log)
    # A growing file, the shape this driver's appendOnly mode exists for. Appended in chunks
    # rather than one write, so a FUSE mount pays per flush the way a real logger makes it.
    for i in $(seq 1 "$FILES"); do
      dd if=/dev/zero bs="$BLOCK" count="$COUNT" status=none >> "$TARGET/app.log"
      WRITTEN=$(( WRITTEN + COUNT * BLOCK ))
    done
    ;;

  write-then-exit)
    # One file, written and then the writer is gone immediately. This is the case a sweep-based
    # design either handles or does not, and the one a preStop hook races with.
    dd if=/dev/zero of="$TARGET/dump.bin" bs="$BLOCK" count="$COUNT" status=none
    WRITTEN=$(( COUNT * BLOCK ))
    ;;

  *)
    echo "unknown shape: $SHAPE" >&2
    exit 2
    ;;
esac

# sync so the comparison is not measuring who left more in the page cache. For s3fs this is
# where the upload actually happens, which is the point: its cost belongs to the writer.
sync

END="$(start_ns)"
echo "WRITER wall_ms=$(( (END - START) / 1000000 )) bytes=$WRITTEN files=$FILES"

#!/usr/bin/env bash
#
# Head-to-head: csi-s3-archiver against s3fs, on one machine, against one S3 endpoint, with
# one writer.
#
#   bench/head-to-head/run.sh --binary target/csi-s3-archiver
#
# WHAT IS HELD CONSTANT. Every arm runs in a container on the same Docker network, writes with
# the same writer.sh, and targets the same SeaweedFS. The local and archiver arms write to the
# same bind-mounted host directory, so the disk underneath them is identical. The only variable
# is what sits between the writer and the bucket.
#
#   local      a plain directory. Nothing is uploaded. The control.
#   s3fs       an s3fs-fuse mount of the bucket. The writer's writes go to S3.
#   archiver   a plain directory that csi-s3-archiver is archiving concurrently.
#
# WHY THE CONTROL ARM EXISTS. Without it, "the archiver is faster than s3fs" is unfalsifiable:
# the archiver writes to local disk, so of course it is. The control says how much of the
# difference is the disk and how much is the design. The archiver's write-path number should
# land on top of the control's, and if it does not, the driver is in the write path somewhere
# it should not be.
#
# TWO COLUMNS, NOT ONE. "Write time" and "time until every byte is in the bucket" are different
# questions and the arms win different ones. Reporting only the first flatters this project;
# reporting only the second flatters s3fs. Both are printed for every arm.
#
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$REPO_ROOT"

BINARY=""
SEAWEED_IMAGE="${SEAWEED_IMAGE:-chrislusf/seaweedfs:4.40}"
S3FS_IMAGE="csi-s3-archiver-bench-s3fs"
S3FS_VERSION=""   # read from the built image; see below
BUCKET="headtohead"
ACCESS_KEY="bench-key"
SECRET_KEY="bench-secret"
RESULTS="$HERE/results.md"
KEEP=0

usage() {
    cat <<'EOF'
Usage: bench/head-to-head/run.sh [options]

Compares csi-s3-archiver against s3fs and against a plain local directory, on one machine,
against one S3 endpoint, with one writer.

Options:
  --binary PATH   the native driver binary (default: target/csi-s3-archiver)
  --results FILE  where to write the report (default: bench/head-to-head/results.md)
  --keep          leave the containers running for inspection
  -h, --help      this help

Requires Docker and /dev/fuse. Nothing is installed on the host: s3fs runs in a pinned
container image this script builds.
EOF
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --binary) BINARY="$2"; shift 2 ;;
        --results) RESULTS="$2"; shift 2 ;;
        --keep) KEEP=1; shift ;;
        -h|--help) usage; exit 0 ;;
        *) echo "error: unknown option: $1" >&2; usage >&2; exit 2 ;;
    esac
done

BINARY="${BINARY:-$REPO_ROOT/target/csi-s3-archiver}"

# --- preconditions ------------------------------------------------------------------------
command -v docker >/dev/null 2>&1 || { echo "error: docker is required" >&2; exit 1; }

if [[ ! -x "$BINARY" ]]; then
    echo "error: $BINARY is not executable. Run ./build-native.sh first." >&2
    echo "       The comparison measures the binary that ships, not the JVM build." >&2
    exit 1
fi

# FUSE is the one thing that cannot be worked around from inside a container. Skip with a
# reason rather than fail, the same way the e2e suite handles a kernel without vxlan.
if [[ ! -c /dev/fuse ]]; then
    echo ">> SKIPPING: /dev/fuse is not available on this host, so s3fs cannot mount." >&2
    echo ">> The archiver and local arms would still run, but a one-armed comparison is not" >&2
    echo ">> a comparison. Run this on a host with the fuse module loaded." >&2
    exit 0
fi

NETWORK="h2h-net-$$"
SEAWEED="h2h-seaweed-$$"
WORK_DIR="$(mktemp -d /tmp/h2h-XXXXXX)"
chmod 777 "$WORK_DIR"

cleanup() {
    local status=$?
    if [[ "$KEEP" != 1 ]]; then
        docker rm -f "$SEAWEED" >/dev/null 2>&1 || true
        docker network rm "$NETWORK" >/dev/null 2>&1 || true
        rm -rf "$WORK_DIR"
    else
        echo ">> --keep: network $NETWORK, seaweed $SEAWEED, work dir $WORK_DIR"
    fi
    return $status
}
trap cleanup EXIT

# --- the S3 endpoint ----------------------------------------------------------------------
# SeaweedFS answers any SigV4-signed request with 400 unless an identity is configured, and
# both the AWS SDK and s3fs sign. Same fixture as the rest of this project's harnesses.
cat >"$WORK_DIR/s3.json" <<EOF
{
  "identities": [
    {
      "name": "bench",
      "credentials": [ { "accessKey": "$ACCESS_KEY", "secretKey": "$SECRET_KEY" } ],
      "actions": ["Admin", "Read", "Write", "List", "Tagging"]
    }
  ]
}
EOF

echo ">> building the s3fs image"
docker build -q -f "$HERE/Dockerfile.s3fs" -t "$S3FS_IMAGE" "$HERE" >/dev/null
# Reported rather than asserted: apt installs whatever the base image ships, so the honest
# thing is to read the version back out of the image that will actually run.
S3FS_VERSION="$(docker run --rm "$S3FS_IMAGE" 's3fs --version 2>&1 | head -1' \
    | sed -n 's/.*V\([0-9.]*\).*/\1/p')"
echo ">>   s3fs-fuse $S3FS_VERSION"

echo ">> starting SeaweedFS ($SEAWEED_IMAGE)"
docker network create "$NETWORK" >/dev/null
docker run -d --name "$SEAWEED" --network "$NETWORK" -P \
    -v "$WORK_DIR/s3.json":/etc/s3.json:ro \
    "$SEAWEED_IMAGE" server -s3 -dir=/data -ip=0.0.0.0 -s3.config=/etc/s3.json >/dev/null

S3_PORT="$(docker port "$SEAWEED" 8333/tcp | head -1 | sed 's/.*://')"
S3_HOST_URL="http://127.0.0.1:$S3_PORT"
S3_NET_URL="http://$SEAWEED:8333"

for _ in $(seq 1 120); do
    curl -s -o /dev/null --max-time 2 "$S3_HOST_URL/" 2>/dev/null && break
    sleep 1
done
echo ">>   endpoint: $S3_HOST_URL (in-network: $S3_NET_URL)"

# s3fs refuses to mount a bucket that does not exist, and SeaweedFS creates one on the first
# PUT. The driver's own upload check is the shortest way to make that PUT, and it doubles as a
# proof that the endpoint and credentials work before any arm is timed.
echo ">> priming the bucket"
S3A_ENDPOINT="$S3_HOST_URL" S3A_BUCKET="$BUCKET" S3A_PATH_STYLE=true \
S3A_PREFIX="prime/" \
S3A_ACCESS_KEY_ID="$ACCESS_KEY" S3A_SECRET_ACCESS_KEY="$SECRET_KEY" \
    "$BINARY" check-upload >"$WORK_DIR/prime.log" 2>&1 || {
        echo "error: could not reach the S3 endpoint with the driver:" >&2
        sed 's/^/   | /' "$WORK_DIR/prime.log" >&2
        exit 1
    }

# --- helpers ------------------------------------------------------------------------------

now_ms() { echo $(( $(date +%s%N) / 1000000 )); }

# Every result row lands here as: shape arm write_ms durable_ms bytes peak_rss_kb
: >"$WORK_DIR/rows.txt"

record() {
    echo "$1 $2 $3 $4 $5 $6" >>"$WORK_DIR/rows.txt"
    printf '  %-16s %-9s write %7s ms   durable %8s ms   %6s MiB\n' \
        "$1" "$2" "$3" "$4" "$(( $5 / 1048576 ))"
}

# Peak RSS of a pid, sampled until it exits. VmHWM is the kernel's own high-water mark, so
# the peak is exact even between samples; the loop exists only to read it before the process
# is gone, which matters because the sweep is the expensive part.
sample_peak_rss() {
    local pid="$1" out="$2"
    echo 0 >"$out"
    while kill -0 "$pid" 2>/dev/null; do
        local hwm
        hwm="$(awk '/^VmHWM:/ {print $2}' "/proc/$pid/status" 2>/dev/null || true)"
        [[ -n "$hwm" ]] && echo "$hwm" >"$out"
        sleep 0.1
    done
}

# --- the arms -----------------------------------------------------------------------------

# local: a plain directory. No upload, so "durable" is not applicable and is reported as -1.
run_local() {
    local shape="$1" files="$2" bytes="$3"
    local dir="$WORK_DIR/local-$shape"
    mkdir -p "$dir"

    local out
    out="$(docker run --rm --network "$NETWORK" -v "$dir":/data "$S3FS_IMAGE" \
        "/usr/local/bin/writer.sh $shape /data $files $bytes")"
    local write_ms bytes_written
    write_ms="$(sed -n 's/.*wall_ms=\([0-9]*\).*/\1/p' <<<"$out")"
    bytes_written="$(sed -n 's/.*bytes=\([0-9]*\).*/\1/p' <<<"$out")"
    record "$shape" "local" "$write_ms" "-1" "$bytes_written" "0"
}

# s3fs: the writer writes straight into the mount, so the upload is inside its wall time.
run_s3fs() {
    local shape="$1" files="$2" bytes="$3"
    local prefix="s3fs-$shape"

    # -o use_path_request_style for a container-hosted endpoint with no per-bucket DNS, and
    # nocopyapi/nomultipart off: the defaults are what a user would get.
    local script="
        echo '$ACCESS_KEY:$SECRET_KEY' > /etc/passwd-s3fs && chmod 600 /etc/passwd-s3fs
        mkdir -p /mnt/s3
        s3fs $BUCKET /mnt/s3 \
            -o url=$S3_NET_URL \
            -o use_path_request_style \
            -o passwd_file=/etc/passwd-s3fs \
            -o dbglevel=crit \
            -o allow_other
        mkdir -p /mnt/s3/$prefix
        /usr/local/bin/writer.sh $shape /mnt/s3/$prefix $files $bytes
        umount /mnt/s3
    "
    local start out
    start="$(now_ms)"
    out="$(docker run --rm --network "$NETWORK" \
        --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \
        "$S3FS_IMAGE" "$script")"
    local write_ms bytes_written
    write_ms="$(sed -n 's/.*wall_ms=\([0-9]*\).*/\1/p' <<<"$out")"
    bytes_written="$(sed -n 's/.*bytes=\([0-9]*\).*/\1/p' <<<"$out")"

    # For s3fs the bytes are in the bucket when the writer returns (the unmount above forces
    # any remainder), so durability costs nothing beyond the write.
    record "$shape" "s3fs" "$write_ms" "$write_ms" "$bytes_written" "0"
}

# archiver: the writer writes to a plain directory that the driver is archiving CONCURRENTLY,
# which is what happens in a pod. The sweep at the end is pod termination.
run_archiver() {
    local shape="$1" files="$2" bytes="$3"
    local dir="$WORK_DIR/arch-$shape"
    local sentinel="$WORK_DIR/done-$shape"
    local prefix="archiver-$shape/"
    mkdir -p "$dir"
    rm -f "$sentinel"

    local append_only="false" strategy="rewrite"
    if [[ "$shape" == "append-log" ]]; then
        append_only="true"
        strategy="segments"
    fi

    local start
    start="$(now_ms)"

    # The driver first, so archiving overlaps the writing exactly as it would in a pod.
    S3A_ENDPOINT="$S3_HOST_URL" \
    S3A_BUCKET="$BUCKET" \
    S3A_PATH_STYLE=true \
    S3A_PREFIX="$prefix" \
    S3A_ACCESS_KEY_ID="$ACCESS_KEY" \
    S3A_SECRET_ACCESS_KEY="$SECRET_KEY" \
    S3A_QUIESCENCE_SECONDS=1 \
    S3A_BENCH_SOURCE_DIR="$dir" \
    S3A_BENCH_UNTIL_FILE="$sentinel" \
    S3A_BENCH_APPEND_ONLY="$append_only" \
    S3A_BENCH_APPEND_STRATEGY="$strategy" \
        "$BINARY" bench >"$WORK_DIR/arch-$shape.log" 2>&1 &
    local driver_pid=$!
    sample_peak_rss "$driver_pid" "$WORK_DIR/rss-$shape" &
    local sampler_pid=$!
    sleep 0.5

    local out
    out="$(docker run --rm --network "$NETWORK" -v "$dir":/data "$S3FS_IMAGE" \
        "/usr/local/bin/writer.sh $shape /data $files $bytes")"
    local write_ms bytes_written
    write_ms="$(sed -n 's/.*wall_ms=\([0-9]*\).*/\1/p' <<<"$out")"
    bytes_written="$(sed -n 's/.*bytes=\([0-9]*\).*/\1/p' <<<"$out")"

    # The writer is gone: this is the pod terminating, and the sweep is what unpublish does.
    touch "$sentinel"
    wait "$driver_pid" || {
        echo "error: the archiver arm failed for $shape:" >&2
        sed 's/^/   | /' "$WORK_DIR/arch-$shape.log" >&2
        exit 1
    }
    local durable_ms=$(( $(now_ms) - start ))
    wait "$sampler_pid" 2>/dev/null || true
    local rss
    rss="$(cat "$WORK_DIR/rss-$shape" 2>/dev/null || echo 0)"

    record "$shape" "archiver" "$write_ms" "$durable_ms" "$bytes_written" "$rss"
}

# --- run ----------------------------------------------------------------------------------
# Shapes chosen to cover what the two designs actually differ on: per-file overhead, one big
# transfer, an append pattern, and a writer that exits immediately.
#                shape             files  bytes-each
SHAPES=(
    "many-small       200    262144"
    "one-large          1 268435456"
    "append-log        64   1048576"
    "write-then-exit    1  67108864"
)

echo
echo ">> running (one writer, three arms, same endpoint)"
echo
for spec in "${SHAPES[@]}"; do
    read -r shape files bytes <<<"$spec"
    echo "  --- $shape ($files x $(( bytes / 1024 )) KiB) ---"
    run_local    "$shape" "$files" "$bytes"
    run_s3fs     "$shape" "$files" "$bytes"
    run_archiver "$shape" "$files" "$bytes"
    echo
done

# --- report -------------------------------------------------------------------------------
# The raw rows are kept beside the report: regenerating the prose should never require
# re-measuring, and a reader can check the arithmetic.
cp "$WORK_DIR/rows.txt" "$HERE/results.tsv"
"$HERE/report.sh" "$HERE/results.tsv" "$S3FS_VERSION" "$SEAWEED_IMAGE" >"$RESULTS"
echo ">> wrote $RESULTS (raw rows in $HERE/results.tsv)"