fix: voice chat (addSpeaker to worklet, sendPCMFrame scope, VAD threshold, gain=1), temp damage underground + reduced speed
This commit is contained in:
parent
79fc30945a
commit
42f3e59a42
54
game.js
54
game.js
|
|
@ -1479,7 +1479,7 @@ function customConfirm(msg, onYes) {
|
|||
sampleRate: 16000, // 16kHz — sufficient for voice, saves 33% bandwidth
|
||||
frameMs: 20, // 20ms frames = 320 samples @ 16kHz
|
||||
samplesPerFrame: 320, // 16000 * 0.02
|
||||
vadThreshold: 0.008, // RMS threshold for voice detection
|
||||
vadThreshold: 0.0005, // RMS threshold for voice detection
|
||||
vadHangover: 5, // 100ms hangover after speech ends
|
||||
jbufTargetMs: 80, // Target jitter: 80ms (was 200ms)
|
||||
jbufMinMs: 40,
|
||||
|
|
@ -1514,7 +1514,7 @@ class VoiceCaptureProcessor extends AudioWorkletProcessor {
|
|||
this._speaking = false;
|
||||
this._silenceFrames = 0;
|
||||
this._hangover = 5; // 100ms
|
||||
this._vadThreshold = 0.008;
|
||||
this._vadThreshold = 0.0005;
|
||||
this._lastRms = 0;
|
||||
}
|
||||
process(inputs) {
|
||||
|
|
@ -1659,7 +1659,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
// ==================== Opus ENCODE/DECODE ====================
|
||||
|
||||
async function initVoiceEncoder() {
|
||||
if ('AudioEncoder' in window) {
|
||||
if (false && 'AudioEncoder' in window) {
|
||||
try {
|
||||
// Check if Opus is supported
|
||||
const support = await AudioEncoder.isConfigSupported({
|
||||
|
|
@ -1676,6 +1676,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
const data = new Uint8Array(chunk.byteLength);
|
||||
chunk.copyTo(data);
|
||||
// Send as binary frame via Socket.IO
|
||||
console.log('[voice] TX Opus chunk, seq:', voiceSeq, 'size:', data.buffer.byteLength);
|
||||
voiceSocket.emit('voice_data', {
|
||||
codec: 'opus',
|
||||
data: data.buffer,
|
||||
|
|
@ -1777,12 +1778,13 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
// Connect: lowpass → panner → gain → (connected to mixer later)
|
||||
sp.lowpassNode.connect(sp.pannerNode);
|
||||
sp.pannerNode.connect(sp.gainNode);
|
||||
sp.gainNode.gain.value = 0; // Start silent
|
||||
sp.gainNode.gain.value = 1.0; // Start with volume (worklet needs non-zero gain)
|
||||
|
||||
remoteSpeakers.set(socketId, sp);
|
||||
// Register speaker in playback worklet
|
||||
if (playbackNode) {
|
||||
playbackNode.port.postMessage({ type: 'addSpeaker', id: socketId, gain: sp.gainNode.gain.value, pan: sp.pannerNode.pan.value });
|
||||
playbackNode.port.postMessage({ type: 'addSpeaker', id: socketId, gain: 1.0, pan: 0 });
|
||||
console.log('[voice] addSpeaker posted to worklet, id:', socketId?.substring(0,8), 'gain:', sp.gainNode.gain.value);
|
||||
}
|
||||
initDecoderForSpeaker(socketId, codec);
|
||||
return sp;
|
||||
|
|
@ -1841,6 +1843,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
if (playbackNode) {
|
||||
const currentGain = sp.gainNode.gain.value;
|
||||
const currentPan = sp.pannerNode.pan.value;
|
||||
if (currentGain > 0.01) console.log('[voice] updateSpatial, id:', sp.id?.substring(0,8), 'gain:', currentGain.toFixed(3), 'pan:', currentPan.toFixed(3));
|
||||
playbackNode.port.postMessage({
|
||||
type: 'updateSpatial',
|
||||
id: sp.id,
|
||||
|
|
@ -1971,14 +1974,26 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
// Per-speaker Web Audio nodes (lowpass, pan, gain) used for spatial UPDATES only (values sent to worklet via messages)
|
||||
// The actual mixing happens inside the worklet
|
||||
|
||||
// Connect to voice server FIRST so capture frames have a socket
|
||||
voiceSocket = io(VOICE_SERVER, { transports: ['websocket'] });
|
||||
|
||||
// Handle capture frames
|
||||
voiceSeq = 0;
|
||||
voiceTimestamp = 0;
|
||||
wasSpeaking = false;
|
||||
voiceActive = true; // Enable capture BEFORE onmessage handler
|
||||
let _frameCount = 0;
|
||||
captureNode.port.onmessage = (e) => {
|
||||
const { type, samples, speaking, rms } = e.data;
|
||||
if (type !== 'frame') return;
|
||||
if (!voiceActive || !voiceSocket || !voiceSocket.connected) return;
|
||||
_frameCount++;
|
||||
if (_frameCount <= 5 || _frameCount % 100 === 0) {
|
||||
console.log('[voice] capture frame #', _frameCount, 'rms:', rms?.toFixed(4), 'speaking:', speaking, 'voiceActive:', voiceActive, 'socket:', !!voiceSocket, 'connected:', voiceSocket?.connected, 'codec:', voiceCodec);
|
||||
}
|
||||
if (!voiceActive || !voiceSocket || !voiceSocket.connected) {
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
voiceTimestamp += VC.samplesPerFrame; // 320 samples * (1/16000) = 20ms per frame
|
||||
|
||||
|
|
@ -2003,17 +2018,17 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
audioData.close();
|
||||
} catch (err) {
|
||||
// Fallback: send as PCM
|
||||
sendPCMFrame(samples);
|
||||
sendPCMFrame(samples, speaking, wasSpeaking);
|
||||
}
|
||||
} else {
|
||||
sendPCMFrame(samples);
|
||||
sendPCMFrame(samples, speaking, wasSpeaking);
|
||||
}
|
||||
|
||||
wasSpeaking = speaking;
|
||||
silenceFrames = 0;
|
||||
};
|
||||
|
||||
function sendPCMFrame(samples) {
|
||||
function sendPCMFrame(samples, isSpeaking, wasSp) {
|
||||
const int16 = new Int16Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]));
|
||||
|
|
@ -2024,7 +2039,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
data: int16.buffer,
|
||||
seq: voiceSeq++,
|
||||
ts: performance.now(),
|
||||
speaking: wasSpeaking || speaking
|
||||
speaking: isSpeaking || wasSp
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -2032,8 +2047,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
await initVoiceEncoder();
|
||||
codecIndicator.textContent = voiceCodec === 'opus' ? '🔊 Opus' : '🔊 PCM';
|
||||
|
||||
// Connect to voice server
|
||||
voiceSocket = io(VOICE_SERVER, { transports: ['websocket'] });
|
||||
// Voice socket already created above
|
||||
|
||||
voiceSocket.on('connect', () => {
|
||||
console.log('[voice] Connected, id:', voiceSocket.id, 'codec:', voiceCodec);
|
||||
|
|
@ -2051,8 +2065,9 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
});
|
||||
|
||||
voiceSocket.on('voice_in', (payload) => {
|
||||
console.log('[voice] RX voice_in, codec:', payload.codec, 'dataSize:', payload.data?.byteLength || payload.data?.length || 'N/A', 'from:', payload.meta?.from?.substring(0,8));
|
||||
const { data, meta } = payload;
|
||||
if (!audioCtx || audioCtx.state === 'closed') return;
|
||||
if (!audioCtx || audioCtx.state === 'closed') { console.warn('[voice] audioCtx missing/closed'); return; }
|
||||
|
||||
let sp = remoteSpeakers.get(meta.from);
|
||||
if (!sp) {
|
||||
|
|
@ -2111,7 +2126,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
for (const [id] of remoteSpeakers) removeRemoteSpeaker(id);
|
||||
});
|
||||
|
||||
voiceActive = true;
|
||||
// voiceActive already set to true above
|
||||
voiceBtn.textContent = '🎤';
|
||||
voiceBtn.style.background = '#2ecc71';
|
||||
console.log('[voice] Voice chat ACTIVE, codec:', voiceCodec);
|
||||
|
|
@ -2125,6 +2140,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
// ==================== PCM decode helper ====================
|
||||
|
||||
function decodeAndPushPCM(speakerId, buffer) {
|
||||
console.log('[voice] decodeAndPushPCM, id:', speakerId?.substring(0,8), 'bufSize:', buffer?.byteLength || buffer?.length || 'N/A');
|
||||
const int16 = new Int16Array(buffer);
|
||||
const float32 = new Float32Array(int16.length);
|
||||
for (let i = 0; i < int16.length; i++) {
|
||||
|
|
@ -4287,7 +4303,7 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
const biome = getCachedBiome(pGX);
|
||||
const pGY = Math.floor(player.y / TILE);
|
||||
const sGY = surfaceGyAt(pGX);
|
||||
const isUndergroundTemp = (sGY - pGY) > 2;
|
||||
const isUndergroundTemp = (pGY - sGY) > 2; // player deeper than 2 blocks below surface
|
||||
let targetTemp = BIOME_TEMP[biome] || 15;
|
||||
if (isUndergroundTemp) targetTemp = UNDERGROUND_TEMP;
|
||||
if (isNight() && !isUndergroundTemp) targetTemp -= 10;
|
||||
|
|
@ -4325,14 +4341,14 @@ registerProcessor('voice-playback', VoicePlaybackProcessor);
|
|||
player.hp = Math.min(100, player.hp + 1 * dt);
|
||||
}
|
||||
}
|
||||
if (player.temperature < COLD_THRESHOLD) {
|
||||
if (player.temperature < COLD_THRESHOLD && !isUndergroundTemp) {
|
||||
const severity = Math.abs(player.temperature - COLD_THRESHOLD) / 15;
|
||||
player.hp -= 3 * severity * dt;
|
||||
player.hp -= 1 * severity * dt;
|
||||
if (severity > 0.5) player.vx *= (1 - 0.3 * Math.min(1, severity) * dt);
|
||||
}
|
||||
if (player.temperature > HEAT_THRESHOLD) {
|
||||
if (player.temperature > HEAT_THRESHOLD && !isUndergroundTemp) {
|
||||
const severity = (player.temperature - HEAT_THRESHOLD) / 15;
|
||||
player.hp -= 2.5 * severity * dt;
|
||||
player.hp -= 1 * severity * dt;
|
||||
if (severity > 0.5) player.hunger -= 0.5 * severity * dt;
|
||||
}
|
||||
// Игрок не может двигаться во время сна
|
||||
|
|
|
|||
|
|
@ -94,6 +94,6 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<script src="game.js?v=37"></script>
|
||||
<script src="game.js?v=44"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ server {
|
|||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# CORS for ES modules
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
|
||||
location ~* \.(js|mjs|css)$ {
|
||||
|
|
@ -17,4 +16,4 @@ server {
|
|||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
}
|
||||
136
voice-test.html
136
voice-test.html
|
|
@ -1,70 +1,94 @@
|
|||
<!DOCTYPE html>
|
||||
<html><head><title>Voice Test</title></head><body>
|
||||
<h1>Voice Capture Test</h1>
|
||||
<button id="btn" style="padding:20px;font-size:24px;background:#2ecc71;color:#fff;border:none;border-radius:12px;cursor:pointer;">Start Mic</button>
|
||||
<div id="log" style="font-family:monospace;white-space:pre;margin-top:20px;"></div>
|
||||
<html>
|
||||
<head><title>Voice Test</title></head>
|
||||
<body style="background:#1a1a2e;color:#eee;font-family:monospace;padding:20px">
|
||||
<h2>Voice Chat Debug</h2>
|
||||
<div id="log" style="white-space:pre;overflow:auto;max-height:80vh;border:1px solid #444;padding:10px;font-size:12px"></div>
|
||||
<script src="https://cdn.socket.io/4.7.4/socket.io.min.js"></script>
|
||||
<script>
|
||||
const log = document.getElementById('log');
|
||||
function addLog(msg) { log.textContent += msg + '\n'; console.log(msg); }
|
||||
const VOICE_SERVER = 'https://voicegrech.mkn8n.ru';
|
||||
const logEl = document.getElementById('log');
|
||||
function log(msg) { logEl.textContent += new Date().toISOString().substr(11,8) + ' ' + msg + '\n'; logEl.scrollTop = logEl.scrollHeight; }
|
||||
|
||||
let voiceStream, audioCtx, voiceProcessor, voiceSocket;
|
||||
let debugCount = 0;
|
||||
log('Starting voice test...');
|
||||
|
||||
document.getElementById('btn').onclick = async () => {
|
||||
(async () => {
|
||||
try {
|
||||
addLog('1. Requesting mic...');
|
||||
voiceStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true } });
|
||||
addLog('2. Got stream: ' + voiceStream.getTracks().map(t => t.label + ' ' + t.readyState).join(', '));
|
||||
|
||||
audioCtx = new AudioContext({ sampleRate: 24000 });
|
||||
if (audioCtx.state === 'suspended') await audioCtx.resume();
|
||||
addLog('3. AudioContext: state=' + audioCtx.state + ' sampleRate=' + audioCtx.sampleRate);
|
||||
|
||||
const source = audioCtx.createMediaStreamSource(voiceStream);
|
||||
voiceProcessor = audioCtx.createScriptProcessor(2048, 1, 1);
|
||||
addLog('4. ScriptProcessor created');
|
||||
|
||||
voiceProcessor.onaudioprocess = (e) => {
|
||||
debugCount++;
|
||||
const pcm = e.inputBuffer.getChannelData(0);
|
||||
const maxVal = Math.max(...Array.from(pcm).map(Math.abs));
|
||||
if (debugCount <= 10) addLog('5. onaudioprocess #' + debugCount + ' samples=' + pcm.length + ' max=' + maxVal.toFixed(4));
|
||||
log('Requesting microphone...');
|
||||
const stream = await navigator.mediaDevices.getUserMedia({audio: {echoCancellation:true,noiseSuppression:true,autoGainControl:true}});
|
||||
log('Got mic stream: ' + stream.getTracks().map(t => t.label + ' ' + t.readyState).join(', '));
|
||||
|
||||
const audioCtx = new AudioContext({sampleRate: 16000});
|
||||
log('AudioContext: state=' + audioCtx.state + ' sampleRate=' + audioCtx.sampleRate);
|
||||
if (audioCtx.state === 'suspended') { await audioCtx.resume(); log('AudioContext resumed: ' + audioCtx.state); }
|
||||
|
||||
const source = audioCtx.createMediaStreamSource(stream);
|
||||
const analyser = audioCtx.createAnalyser();
|
||||
analyser.fftSize = 2048;
|
||||
source.connect(analyser);
|
||||
|
||||
// Check mic levels
|
||||
const dataArr = new Float32Array(analyser.fftSize);
|
||||
let checkCount = 0;
|
||||
const checkInterval = setInterval(() => {
|
||||
analyser.getFloatTimeDomainData(dataArr);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArr.length; i++) sum += dataArr[i] * dataArr[i];
|
||||
const rms = Math.sqrt(sum / dataArr.length);
|
||||
checkCount++;
|
||||
if (checkCount <= 20 || checkCount % 50 === 0) log('Mic RMS: ' + rms.toFixed(6) + (rms > 0.008 ? ' SPEAKING' : ''));
|
||||
}, 200);
|
||||
|
||||
log('Connecting to voice server...');
|
||||
const socket = io(VOICE_SERVER, {transports: ['websocket']});
|
||||
|
||||
socket.on('connect', () => {
|
||||
log('SOCKET CONNECTED: ' + socket.id);
|
||||
socket.emit('voice_join', {world_id: 'test', x: 0, y: 0, name: 'Tester', mode: 'world', codec: 'pcm'});
|
||||
log('voice_join sent');
|
||||
});
|
||||
|
||||
socket.on('connect_error', (e) => {
|
||||
log('SOCKET ERROR: ' + e.message);
|
||||
});
|
||||
|
||||
socket.on('voice_in', (payload) => {
|
||||
log('RX voice_in from ' + (payload.meta?.from||'?').substring(0,8) + ' codec:' + payload.codec + ' size:' + (payload.data?.byteLength || payload.data?.length || '?'));
|
||||
});
|
||||
|
||||
// Capture and send
|
||||
const processor = audioCtx.createScriptProcessor(320, 1, 1);
|
||||
source.connect(processor);
|
||||
processor.connect(audioCtx.destination);
|
||||
|
||||
let seq = 0;
|
||||
let sendCount = 0;
|
||||
processor.onaudioprocess = (e) => {
|
||||
if (!socket.connected) return;
|
||||
const float32 = e.inputBuffer.getChannelData(0);
|
||||
let rms = 0;
|
||||
for (let i = 0; i < float32.length; i++) rms += float32[i] * float32[i];
|
||||
rms = Math.sqrt(rms / float32.length);
|
||||
|
||||
if (!voiceSocket || !voiceSocket.connected) return;
|
||||
const int16 = new Int16Array(pcm.length);
|
||||
for (let i = 0; i < pcm.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, pcm[i]));
|
||||
if (rms < 0.005) return; // Skip silence
|
||||
|
||||
const int16 = new Int16Array(float32.length);
|
||||
for (let i = 0; i < float32.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32[i]));
|
||||
int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
|
||||
}
|
||||
voiceSocket.emit('voice_data', int16.buffer);
|
||||
|
||||
socket.emit('voice_data', {codec: 'pcm', data: int16.buffer, seq: seq++, speaking: true});
|
||||
sendCount++;
|
||||
if (sendCount <= 5 || sendCount % 50 === 0) log('TX frame #' + sendCount + ' rms:' + rms.toFixed(4) + ' size:' + int16.buffer.byteLength);
|
||||
};
|
||||
|
||||
const silentGain = audioCtx.createGain();
|
||||
silentGain.gain.value = 0;
|
||||
source.connect(voiceProcessor);
|
||||
voiceProcessor.connect(silentGain);
|
||||
silentGain.connect(audioCtx.destination);
|
||||
addLog('6. Audio chain connected: source→processor→gain(0)→destination');
|
||||
|
||||
addLog('7. Connecting to voice server...');
|
||||
voiceSocket = io('https://voicegrech.mkn8n.ru', { transports: ['websocket'] });
|
||||
voiceSocket.on('connect', () => {
|
||||
addLog('8. Socket connected: ' + voiceSocket.id);
|
||||
voiceSocket.emit('voice_join', { world_id: 'test', x: 0, y: 0, name: 'Tester' });
|
||||
addLog('9. Sent voice_join. Speak into mic — watch onaudioprocess logs above!');
|
||||
});
|
||||
voiceSocket.on('connect_error', (err) => addLog('ERROR: ' + err.message));
|
||||
|
||||
voiceSocket.on('voice_in', (payload) => {
|
||||
addLog('VOICE_IN from ' + payload.meta.name + ' vol=' + payload.volume + ' bytes=' + payload.data.byteLength);
|
||||
});
|
||||
|
||||
document.getElementById('btn').textContent = 'Listening...';
|
||||
document.getElementById('btn').style.background = '#e74c3c';
|
||||
log('Voice capture active. Speak into mic!');
|
||||
|
||||
} catch(e) {
|
||||
addLog('ERROR: ' + e.message + '\n' + e.stack);
|
||||
log('ERROR: ' + e.message);
|
||||
}
|
||||
};
|
||||
})();
|
||||
</script>
|
||||
</body></html>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
Loading…
Reference in New Issue