{ "$schema": "https://schema.org/Dataset", "@context": "https://schema.org", "@type": "Dataset", "@id": "https://multilarm.com/benchmarks.json#dataset", "name": "Multilarm measured performance", "description": "Measured performance figures for Multilarm on its reference device, a Raspberry Pi 400. Every figure states the device, the method, the date and the command that reproduces it. Figures that have regressed are kept in the history, and things not measured are listed rather than omitted.", "url": "https://multilarm.com/data/", "sameAs": "https://multilarm.com/data/", "license": "https://multilarm.com/data/", "isAccessibleForFree": true, "creator": { "@id": "https://multilarm.com/#organization" }, "publisher": { "@id": "https://multilarm.com/#organization" }, "dateModified": "2026-09-23", "measurementTechnique": "On-device benchmarking via ops/bench/bench.mjs over the device REST API, plus the built-in device self-test battery.", "variableMeasured": [ { "@type": "PropertyValue", "name": "Neural TTS real-time factor, shipped uint8 model (Raspberry Pi 400)", "value": 2.14, "description": "Warm cache, 95-character line, Kokoro 82M uint8 ONNX under ONNX Runtime with the sequential executor." }, { "@type": "PropertyValue", "name": "Neural TTS real-time factor, fp32 model (Raspberry Pi 400)", "value": 2.49, "description": "Same line and conditions as the uint8 measurement, unquantised model, 325 MB." }, { "@type": "PropertyValue", "name": "Neural TTS real-time factor, int8 dynamic quantisation (Raspberry Pi 400)", "value": 5, "description": "Dynamic per-tensor quantisation, 92 MB." }, { "@type": "PropertyValue", "name": "Time to first audio, cold utterance (Raspberry Pi 400)", "value": 5, "unitText": "seconds", "description": "Clause-split streaming: the first clause begins playing while the remainder is still being synthesised." }, { "@type": "PropertyValue", "name": "Time to first audio, pre-warmed or repeated line (Raspberry Pi 400)", "value": 4.15, "unitText": "seconds", "description": "Disk cache hit following keystroke-triggered pre-warm, measured on the device." }, { "@type": "PropertyValue", "name": "Cloud round trip from the device (Raspberry Pi 400)", "value": 21, "unitText": "milliseconds", "description": "Device self-test D-D01, measured on the production Pi against the live cloud service." }, { "@type": "PropertyValue", "name": "Status push interval (Raspberry Pi 400)", "value": 30, "unitText": "seconds", "description": "Fixed interval, plus an immediate push after any command." }, { "@type": "PropertyValue", "name": "Command poll interval (Raspberry Pi 400)", "value": 5, "unitText": "seconds", "description": "Fixed interval, backing off exponentially to 60 s while the network is unreachable." }, { "@type": "PropertyValue", "name": "Audio sequence per-item open cost (Raspberry Pi 400)", "value": 14, "unitText": "milliseconds", "description": "Measured on the device against a 150 ms design budget." }, { "@type": "PropertyValue", "name": "Device self-test checks (Raspberry Pi 400)", "value": 39, "unitText": "checks", "description": "Count of distinct checks in the built-in diagnostics battery." }, { "@type": "PropertyValue", "name": "Features audited against production hardware (Raspberry Pi 400)", "value": 156, "unitText": "features", "description": "Every advertised feature exercised on a real device and graded green, amber or red." }, { "@type": "PropertyValue", "name": "Full music library scan, 492 files (Raspberry Pi 400)", "value": 4113, "unitText": "milliseconds", "description": "A cold full scan of 492 audio files totalling 1.3 GB across three folders on the reference Pi, reading ID3, FLAC, Ogg, MP4 and WAV tags and measuring WAV levels. Timed over the device API from POST /api/library/scan?full=1 until the scan reported idle, 2026-09-13." }, { "@type": "PropertyValue", "name": "Incremental rescan, nothing changed (Raspberry Pi 400)", "value": 1088, "unitText": "milliseconds", "description": "The same 492-file library rescanned immediately afterwards with no file changed, timed the same way, 2026-09-13." }, { "@type": "PropertyValue", "name": "LAN listen stream, bandwidth per listener (Raspberry Pi 400)", "value": 1.54, "unitText": "megabits per second", "description": "Measured by pulling /listen.wav from the broadcast server on the reference Pi while the rotation was playing, 2026-09-13: 2,296,364 bytes for 11.96 seconds of audio, 48 kHz 16-bit stereo PCM." } ], "multilarm": { "schemaVersion": 1, "honesty": "These are vendor figures measured on vendor hardware. The harness that produced them ships with the software so that anyone can get their own numbers on their own device; where a figure predates the harness, the method is documented instead. No competitor figures appear here, because we would measure other people’s software badly and nobody should believe us if we did.", "referenceDevice": { "id": "pi400", "name": "Raspberry Pi 400", "cpu": "Broadcom BCM2711, quad-core Cortex-A72 at 1.8 GHz", "isa": "ARMv8.0-A (aarch64)", "memory": "4 GB LPDDR4", "os": "Raspberry Pi OS 64-bit", "role": "reference device - the figures below are what a modest, cheap, fanless machine does, not what a workstation does", "note": "Chosen deliberately as the reference. A benchmark run on fast hardware tells a buyer nothing about the inexpensive board they were going to use." }, "devices": { "pi400": { "id": "pi400", "name": "Raspberry Pi 400", "cpu": "Broadcom BCM2711, quad-core Cortex-A72 at 1.8 GHz", "isa": "ARMv8.0-A (aarch64)", "memory": "4 GB LPDDR4", "os": "Raspberry Pi OS 64-bit", "role": "reference device - the figures below are what a modest, cheap, fanless machine does, not what a workstation does", "note": "Chosen deliberately as the reference. A benchmark run on fast hardware tells a buyer nothing about the inexpensive board they were going to use." } }, "figures": [ { "id": "tts-rtf-uint8", "group": "Speech synthesis", "name": "Neural TTS real-time factor, shipped uint8 model", "value": 2.14, "approximate": false, "unit": "ratio", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-06", "method": "Warm cache, 95-character line, Kokoro 82M uint8 ONNX under ONNX Runtime with the sequential executor.", "meaning": "2.14 seconds of computation per second of speech. The device is not synthesising faster than real time, which is why announcements are streamed clause by clause rather than rendered whole.", "reproduce": "node ops/bench/bench.mjs --host --only tts", "previously": [] }, { "id": "tts-rtf-fp32", "group": "Speech synthesis", "name": "Neural TTS real-time factor, fp32 model", "value": 2.49, "approximate": false, "unit": "ratio", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-06", "method": "Same line and conditions as the uint8 measurement, unquantised model, 325 MB.", "meaning": "About 15% slower and 1.8x larger than the model we ship. Published because it is the obvious alternative and somebody will ask.", "reproduce": "Not automated - requires swapping the model file by hand.", "previously": [] }, { "id": "tts-rtf-int8-dynamic", "group": "Speech synthesis", "name": "Neural TTS real-time factor, int8 dynamic quantisation", "value": 5, "approximate": true, "unit": "ratio", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-06", "method": "Dynamic per-tensor quantisation, 92 MB.", "meaning": "Roughly twice as slow on ARM despite being the smallest model. A negative result, published because the smaller file looks like the obvious win and is not.", "reproduce": "Not automated - requires swapping the model file by hand.", "previously": [] }, { "id": "ttfa-cold", "group": "Speech synthesis", "name": "Time to first audio, cold utterance", "value": 5, "approximate": true, "unit": "seconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-06", "method": "Clause-split streaming: the first clause begins playing while the remainder is still being synthesised.", "meaning": "How long after you press the button before anybody hears anything, for a line the device has never spoken before.", "reproduce": "node ops/bench/bench.mjs --host --only tts", "previously": [ { "value": 28, "unit": "seconds", "date": "2026-05", "note": "Before clause splitting: the whole utterance was synthesised before anything played. Kept because a 5-second figure means nothing without the number it replaced." } ] }, { "id": "ttfa-warm", "group": "Speech synthesis", "name": "Time to first audio, pre-warmed or repeated line", "value": 4.15, "approximate": false, "unit": "seconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-06", "method": "Disk cache hit following keystroke-triggered pre-warm, measured on the device.", "meaning": "Scheduled announcements are pre-warmed, so this is the figure that applies to almost all real traffic.", "reproduce": "node ops/bench/bench.mjs --host --only tts", "previously": [ { "value": 13, "unit": "seconds", "date": "2026-05", "note": "Before the pre-warm cache-key parity fix, warm lines were missing the cache and re-synthesising every time." } ] }, { "id": "cloud-rtt", "group": "Network", "name": "Cloud round trip from the device", "value": 21, "approximate": false, "unit": "milliseconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-08-15", "method": "Device self-test D-D01, measured on the production Pi against the live cloud service.", "meaning": "Relevant only to the optional cloud dashboard. Nothing on the device waits for this: scheduling, playback and emergency output all run with the network unplugged.", "reproduce": "Multilarm --diagnose, or node ops/bench/bench.mjs --host --only selftest", "previously": [] }, { "id": "status-push", "group": "Network", "name": "Status push interval", "value": 30, "approximate": false, "unit": "seconds", "betterIs": "context", "device": "pi400", "measuredOn": "2026-08-15", "method": "Fixed interval, plus an immediate push after any command.", "meaning": "How stale the cloud dashboard can be. Not a performance figure but a design constant, published so nobody has to guess.", "reproduce": "Read the configuration; the interval is not tunable.", "previously": [] }, { "id": "command-poll", "group": "Network", "name": "Command poll interval", "value": 5, "approximate": false, "unit": "seconds", "betterIs": "context", "device": "pi400", "measuredOn": "2026-08-15", "method": "Fixed interval, backing off exponentially to 60 s while the network is unreachable.", "meaning": "Worst-case delay between pressing a button in the cloud dashboard and the device acting on it.", "reproduce": "Read the configuration; the interval is not tunable.", "previously": [] }, { "id": "sequence-open-cost", "group": "Timing precision", "name": "Audio sequence per-item open cost", "value": 14, "approximate": false, "unit": "milliseconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-08-01", "method": "Measured on the device against a 150 ms design budget.", "meaning": "The gap the software adds between two files in a sequence, on top of any silence you asked for. This is the figure that decides whether a bell lands on the minute.", "reproduce": "Multilarm --diagnose (self-test D-A04)", "previously": [] }, { "id": "selftest-checks", "group": "Coverage", "name": "Device self-test checks", "value": 39, "approximate": false, "unit": "checks", "betterIs": "higher", "device": "pi400", "measuredOn": "2026-08-15", "method": "Count of distinct checks in the built-in diagnostics battery.", "meaning": "How much of itself the software can verify without a human. Every one prints its measured value rather than a pass badge.", "reproduce": "Multilarm --diagnose", "previously": [] }, { "id": "feature-audit", "group": "Coverage", "name": "Features audited against production hardware", "value": 156, "approximate": false, "unit": "features", "betterIs": "context", "device": "pi400", "measuredOn": "2026-08-03", "method": "Every advertised feature exercised on a real device and graded green, amber or red.", "meaning": "148 green, 8 amber, 0 red. The amber ones are named in the audit rather than rounded up to green.", "reproduce": "Not automated - the audit is a documented manual pass.", "previously": [] }, { "id": "library-scan-full", "group": "Music library", "name": "Full music library scan, 492 files", "value": 4113, "approximate": false, "unit": "milliseconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-09", "method": "A cold full scan of 492 audio files totalling 1.3 GB across three folders on the reference Pi, reading ID3, FLAC, Ogg, MP4 and WAV tags and measuring WAV levels. Timed over the device API from POST /api/library/scan?full=1 until the scan reported idle, 2026-09-13.", "meaning": "490 files indexed and 2 reported unreadable in a little over four seconds. That is the cold figure, taken with nothing in the page cache; an immediate repeat of the same full pass ran in 3127 ms, and the cold number is the one published because it is the one a first run costs. A full scan is the once-per-setup case either way; it is not on the path of anything a listener hears.", "reproduce": "curl -X POST http://:6580/api/library/scan?full=1 and poll /api/library/status", "previously": [] }, { "id": "library-scan-incremental", "group": "Music library", "name": "Incremental rescan, nothing changed", "value": 1088, "approximate": false, "unit": "milliseconds", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-09", "method": "The same 492-file library rescanned immediately afterwards with no file changed, timed the same way, 2026-09-13.", "meaning": "A quarter of the cost of the full pass, because unchanged files are matched on size and modification time rather than re-read. This is the figure that matters, since it is what a scheduled rescan costs.", "reproduce": "curl -X POST http://:6580/api/library/scan and poll /api/library/status", "previously": [] }, { "id": "lan-listen-bitrate", "group": "Music library", "name": "LAN listen stream, bandwidth per listener", "value": 1.54, "approximate": false, "unit": "megabits per second", "betterIs": "lower", "device": "pi400", "measuredOn": "2026-09", "method": "Measured by pulling /listen.wav from the broadcast server on the reference Pi while the rotation was playing, 2026-09-13: 2,296,364 bytes for 11.96 seconds of audio, 48 kHz 16-bit stereo PCM.", "meaning": "Uncompressed audio costs what uncompressed audio costs. At 32 listeners that is roughly 49 Mbit/s, which is why this is a LAN convenience and not an internet stream. Below 48 kHz the figure falls proportionally - a 44.1 kHz engine gives 1.41 Mbit/s.", "reproduce": "curl -m 12 -o out.wav \"http://:8000/listen.wav?key=\" while background music is playing", "previously": [] } ], "notMeasured": [ { "name": "Neural TTS with int8 static QDQ quantisation", "why": "Not measurable: the quantisation corrupts the model output, so there is no correct result to time. Published as a dead end rather than omitted." }, { "name": "Performance on Windows and on x86-64 Linux", "why": "Both are faster than the reference Pi by a wide margin, but the harness has not been run on a standard configuration often enough to publish a figure. Running it is the fix, not estimating." }, { "name": "Concurrent-device load on the cloud service", "why": "Not yet measured under realistic fleet load. The figure would be invented and it would be flattering, which is exactly why it is absent." } ], "harness": { "path": "ops/bench/bench.mjs", "usage": "node ops/bench/bench.mjs --host --user ", "notes": "Drives a real device over its REST API. There is no simulation mode: a figure produced by a stub is not a figure.", "runs": [] }, "related": { "humanReadable": "https://multilarm.com/data/", "featureMatrix": "https://multilarm.com/features.json", "comparisons": "https://multilarm.com/compare/" } } }