Skip to content

Commit 2bec06f

Browse files
committed
Research: speech survey for seven more languages
Before adding Russian, Dutch, Latin, Tagalog, Japanese, Bosnian and Portuguese: 56 item names (8 per language) spoken by neural voices (Italian voices reading the Latin names, as there is no Latin voice), transcribed by the small model with the language forced, without a prompt and with the expected name as prompt. A separate workflow, so the normal speech check is not affected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Md9vhyasScVaMy34y8HY66
1 parent 987cdd8 commit 2bec06f

3 files changed

Lines changed: 237 additions & 0 deletions

File tree

‎.github/survey/survey.tsv‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Survey of how well Whisper hears item names in languages LangMod might add (.github/workflows/survey.yml).
2+
# file (unused) text the voice says voice (unused) Whisper language code
3+
ru_apple - Яблоко ru-RU-DmitryNeural - ru
4+
ru_dirt - Земля ru-RU-SvetlanaNeural - ru
5+
ru_bread - Хлеб ru-RU-DmitryNeural - ru
6+
ru_torch - Факел ru-RU-SvetlanaNeural - ru
7+
ru_pumpkin - Тыква ru-RU-DmitryNeural - ru
8+
ru_glass - Стекло ru-RU-SvetlanaNeural - ru
9+
ru_diamond_pickaxe - Алмазная кирка ru-RU-DmitryNeural - ru
10+
ru_iron_ingot - Железный слиток ru-RU-SvetlanaNeural - ru
11+
nl_apple - Appel nl-NL-MaartenNeural - nl
12+
nl_dirt - Aarde nl-NL-FennaNeural - nl
13+
nl_bread - Brood nl-NL-MaartenNeural - nl
14+
nl_torch - Fakkel nl-NL-FennaNeural - nl
15+
nl_pumpkin - Pompoen nl-NL-MaartenNeural - nl
16+
nl_glass - Glas nl-NL-FennaNeural - nl
17+
nl_diamond_pickaxe - Diamanten houweel nl-NL-MaartenNeural - nl
18+
nl_iron_ingot - IJzerstaaf nl-NL-FennaNeural - nl
19+
la_apple - Malum it-IT-DiegoNeural - la
20+
la_dirt - Humus it-IT-ElsaNeural - la
21+
la_bread - Panis it-IT-DiegoNeural - la
22+
la_torch - Fax it-IT-ElsaNeural - la
23+
la_pumpkin - Cucurbita it-IT-DiegoNeural - la
24+
la_glass - Vitrum it-IT-ElsaNeural - la
25+
la_diamond_pickaxe - Dolabra adamantina it-IT-DiegoNeural - la
26+
la_iron_ingot - Later ferreus it-IT-ElsaNeural - la
27+
tl_apple - Mansanas fil-PH-AngeloNeural - tl
28+
tl_dirt - Lupa fil-PH-BlessicaNeural - tl
29+
tl_bread - Tinapay fil-PH-AngeloNeural - tl
30+
tl_torch - Sulo fil-PH-BlessicaNeural - tl
31+
tl_pumpkin - Kalabasa fil-PH-AngeloNeural - tl
32+
tl_glass - Salamin fil-PH-BlessicaNeural - tl
33+
tl_diamond_pickaxe - Diyamanteng Piko fil-PH-AngeloNeural - tl
34+
tl_iron_ingot - Linggote ng Bakal fil-PH-BlessicaNeural - tl
35+
ja_apple - リンゴ ja-JP-KeitaNeural - ja
36+
ja_dirt - 土 ja-JP-NanamiNeural - ja
37+
ja_bread - パン ja-JP-KeitaNeural - ja
38+
ja_torch - 松明 ja-JP-NanamiNeural - ja
39+
ja_pumpkin - カボチャ ja-JP-KeitaNeural - ja
40+
ja_glass - ガラス ja-JP-NanamiNeural - ja
41+
ja_diamond_pickaxe - ダイヤモンドのツルハシ ja-JP-KeitaNeural - ja
42+
ja_iron_ingot - 鉄インゴット ja-JP-NanamiNeural - ja
43+
bs_apple - Jabuka bs-BA-GoranNeural - bs
44+
bs_dirt - Zemlja bs-BA-VesnaNeural - bs
45+
bs_bread - Hljeb bs-BA-GoranNeural - bs
46+
bs_torch - Baklja bs-BA-VesnaNeural - bs
47+
bs_pumpkin - Bundeva bs-BA-GoranNeural - bs
48+
bs_glass - Staklo bs-BA-VesnaNeural - bs
49+
bs_diamond_pickaxe - Dijamantski kramp bs-BA-GoranNeural - bs
50+
bs_iron_ingot - Gvozdena poluga bs-BA-VesnaNeural - bs
51+
pt_apple - Maçã pt-BR-AntonioNeural - pt
52+
pt_dirt - Terra pt-BR-FranciscaNeural - pt
53+
pt_bread - Pão pt-BR-AntonioNeural - pt
54+
pt_torch - Tocha pt-BR-FranciscaNeural - pt
55+
pt_pumpkin - Abóbora pt-BR-AntonioNeural - pt
56+
pt_glass - Vidro pt-BR-FranciscaNeural - pt
57+
pt_diamond_pickaxe - Picareta de Diamante pt-BR-AntonioNeural - pt
58+
pt_iron_ingot - Lingote de Ferro pt-BR-FranciscaNeural - pt

‎.github/workflows/survey.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Research: how well Whisper hears item names in languages LangMod might add.
2+
# Not part of the normal checks; it runs when the survey files change or by hand.
3+
name: survey
4+
5+
on:
6+
push:
7+
paths:
8+
- '.github/survey/**'
9+
- 'src/test/java/io/github/dylantopic/langmod/survey/**'
10+
- '.github/workflows/survey.yml'
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: survey-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
samples:
19+
runs-on: ubuntu-24.04
20+
timeout-minutes: 15
21+
steps:
22+
- uses: actions/checkout@v6
23+
- uses: actions/setup-python@v6
24+
with:
25+
python-version: '3.12'
26+
- name: install text-to-speech
27+
run: pip install --quiet edge-tts imageio-ffmpeg
28+
- name: synthesise samples
29+
run: python3 .github/speech/make_samples.py .github/survey/survey.tsv samples
30+
- uses: actions/upload-artifact@v7
31+
with:
32+
name: survey-samples
33+
path: samples/
34+
35+
survey:
36+
needs: samples
37+
runs-on: ubuntu-24.04
38+
timeout-minutes: 30
39+
steps:
40+
- uses: actions/checkout@v6
41+
- uses: actions/setup-java@v5
42+
with:
43+
java-version: '25'
44+
distribution: 'microsoft'
45+
- uses: actions/download-artifact@v7
46+
with:
47+
name: survey-samples
48+
path: samples
49+
- name: cache whisper model
50+
uses: actions/cache@v6
51+
with:
52+
path: models
53+
key: whisper-ggml-small-v1
54+
- name: download whisper model and libraries
55+
run: |
56+
mkdir -p models libs
57+
if [ ! -f models/ggml-small.bin ]; then
58+
curl -sSfL -o models/ggml-small.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin
59+
fi
60+
M=https://repo1.maven.org/maven2
61+
curl -sSfL -o libs/whisper-jni.jar $M/io/github/givimad/whisper-jni/1.7.1/whisper-jni-1.7.1.jar
62+
curl -sSfL -o libs/gson.jar $M/com/google/code/gson/gson/2.13.1/gson-2.13.1.jar
63+
- name: compile
64+
run: |
65+
find src/main/java/io/github/dylantopic/langmod/core -name '*.java' > sources.txt
66+
find src/test/java/io/github/dylantopic/langmod/survey -name '*.java' >> sources.txt
67+
javac -encoding UTF-8 --release 25 -cp "libs/whisper-jni.jar:libs/gson.jar" -d out @sources.txt
68+
- name: transcribe
69+
run: |
70+
java -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 --enable-native-access=ALL-UNNAMED \
71+
-cp "out:libs/whisper-jni.jar:libs/gson.jar" io.github.dylantopic.langmod.survey.SpeechSurvey models/ggml-small.bin samples 2
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package io.github.dylantopic.langmod.survey;
2+
3+
import java.io.IOException;
4+
import java.nio.charset.StandardCharsets;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.text.Normalizer;
8+
import java.util.LinkedHashMap;
9+
import java.util.List;
10+
import java.util.Locale;
11+
import java.util.Map;
12+
13+
import javax.sound.sampled.AudioFormat;
14+
import javax.sound.sampled.AudioInputStream;
15+
import javax.sound.sampled.AudioSystem;
16+
import javax.sound.sampled.UnsupportedAudioFileException;
17+
18+
import io.github.dylantopic.langmod.core.speech.SpeechEngine;
19+
import io.github.dylantopic.langmod.core.speech.WhisperJniEngine;
20+
import io.github.dylantopic.langmod.core.speech.WhisperModels;
21+
22+
/**
23+
* Research tool: how well Whisper hears item names in languages LangMod does not support yet.
24+
* Transcribes .github/survey/survey.tsv without a prompt and with the expected name as prompt,
25+
* and counts transcripts equal to the name once case, accents, punctuation and kana type are ignored.
26+
*
27+
* <p>Usage: SpeechSurvey model.bin samples_dir threads
28+
*/
29+
public final class SpeechSurvey {
30+
private SpeechSurvey() {
31+
}
32+
33+
public static void main(String[] args) throws Exception {
34+
Path model = Path.of(args[0]);
35+
Path samples = Path.of(args[1]);
36+
int threads = Integer.parseInt(args[2]);
37+
Map<String, int[]> stats = new LinkedHashMap<>(); // language -> {plain ok, biased ok, total}
38+
39+
try (SpeechEngine engine = new WhisperJniEngine(model, threads, true, line -> { })) {
40+
engine.load();
41+
42+
for (String line : Files.readAllLines(samples.resolve("samples.tsv"), StandardCharsets.UTF_8)) {
43+
if (line.isBlank() || line.startsWith("#")) {
44+
continue;
45+
}
46+
47+
String[] f = line.split("\t");
48+
String file = f[0];
49+
String text = f[2];
50+
String code = f[5];
51+
float[] audio = readWav(samples.resolve(file + ".wav"));
52+
int context = WhisperModels.audioContext(audio.length, 512);
53+
String plain = engine.transcribe(audio, new SpeechEngine.Request(code, null, context)).trim();
54+
String biased = engine.transcribe(audio, new SpeechEngine.Request(code, text, context)).trim();
55+
boolean plainOk = fold(plain).equals(fold(text));
56+
boolean biasedOk = fold(biased).equals(fold(text));
57+
int[] counts = stats.computeIfAbsent(code, key -> new int[3]);
58+
counts[0] += plainOk ? 1 : 0;
59+
counts[1] += biasedOk ? 1 : 0;
60+
counts[2]++;
61+
System.out.printf(Locale.ROOT, "%-20s want %-24s plain %-28s %s biased %-28s %s%n", file, text, '"' + plain + '"',
62+
plainOk ? "OK" : "--", '"' + biased + '"', biasedOk ? "OK" : "--");
63+
}
64+
}
65+
66+
System.out.println();
67+
System.out.println("language same as the name: no prompt / expected name as prompt");
68+
69+
for (Map.Entry<String, int[]> entry : stats.entrySet()) {
70+
int[] c = entry.getValue();
71+
System.out.printf(Locale.ROOT, "%-8s %d of %d / %d of %d%n", entry.getKey(), c[0], c[2], c[1], c[2]);
72+
}
73+
}
74+
75+
/** Lowercase letters and digits only, without accents, katakana as hiragana. */
76+
static String fold(String text) {
77+
String s = Normalizer.normalize(text, Normalizer.Form.NFKC).toLowerCase(Locale.ROOT);
78+
s = Normalizer.normalize(s, Normalizer.Form.NFD).replaceAll("\\p{M}+", "");
79+
StringBuilder out = new StringBuilder();
80+
81+
for (int cp : s.codePoints().toArray()) {
82+
if (cp >= 0x30A1 && cp <= 0x30F6) {
83+
out.appendCodePoint(cp - 0x60); // katakana -> hiragana
84+
} else if (Character.isLetterOrDigit(cp) || cp == 0x30FC) {
85+
out.appendCodePoint(cp);
86+
}
87+
}
88+
89+
return Normalizer.normalize(out.toString(), Normalizer.Form.NFC);
90+
}
91+
92+
static float[] readWav(Path file) throws IOException, UnsupportedAudioFileException {
93+
try (AudioInputStream in = AudioSystem.getAudioInputStream(file.toFile())) {
94+
AudioFormat format = in.getFormat();
95+
byte[] bytes = in.readAllBytes();
96+
boolean bigEndian = format.isBigEndian();
97+
float[] samples = new float[bytes.length / 2];
98+
99+
for (int i = 0; i < samples.length; i++) {
100+
int lo = bytes[2 * i + (bigEndian ? 1 : 0)] & 0xFF;
101+
int hi = bytes[2 * i + (bigEndian ? 0 : 1)];
102+
samples[i] = (short) ((hi << 8) | lo) / 32768f;
103+
}
104+
105+
return samples;
106+
}
107+
}
108+
}

0 commit comments

Comments
 (0)