使用TensorFlow.js构建DDQN贪吃蛇游戏

DDQN Snake · TensorFlow.js body { background: #1e1e2f; font-family: 'Segoe UI', system-ui, sans-serif; display: flex; justify-content: center; align-items: center; min-height: 100vh; margin: 0; color: #ddd; } .game-container { background: #2a2a3d; padding: 1.5rem; border-radius: 24px; box-shadow: 0 10px 30px rgba(0,0,0,0.5); display: flex; flex-direction: column; align-items: center; gap: 1rem; } canvas { background: #0f0f1a; border-radius: 12px; box-shadow: inset 0 0 10px #00000055; border: 2px solid #3e3e5e; } .status-bar { display: flex; gap: 2rem; align-items: center; background: #1a1a2a; padding: 0.5rem 1.5rem; border-radius: 40px; font-weight: 600; } .status-bar span { color: #b0b0d0; } .btn-group { display: flex; gap: 0.8rem; } button { background: #4a4a6a; border: none; color: white; padding: 0.5rem 1.2rem; border-radius: 40px; font-weight: 600; cursor: pointer; transition: 0.2s; letter-spacing: 0.5px; border: 1px solid #6a6a8a; } button:hover { background: #5f5f85; transform: scale(0.98); } button:disabled { opacity: 0.4; pointer-events: none; } .slider-container { display: flex; align-items: center; gap: 0.5rem; font-size: 0.9rem; } input[type="range"] { width: 140px; background: #3a3a5a; accent-color: #8a8aff; } 🍎 0 🧠 1.00 📦 0 ▶ TRAIN ⏸️ PAUSE 🔄 RESET ⚡ speed 8x // ============================================================ // DDQN SNAKE ， TensorFlow.js (latest API) · single HTML // Double DQN with target network, experience replay, epsilon decay // ============================================================ // ---------- CONFIGURABLE ---------- const canvasWidth = 400; // ${canvasWidth:400} const canvasHeight = 400; // ${canvasHeight:400} const GRID_SIZE = 20; // 20x20 grid const NUM_ACTIONS = 4; // 0:up, 1:down, 2:left, 3:right const HIDDEN_SIZE = 128; const BATCH_SIZE = 64; const GAMMA = 0.95; const LEARNING_RATE = 0.001; const MEMORY_CAPACITY = 20000; const TARGET_UPDATE_FREQ = 200; // steps const MIN_EPSILON = 0.05; const DECAY_RATE = 0.9995; // ---------- GAME STATE ---------- let canvas = document.getElementById('gameCanvas'); let ctx = canvas.getContext('2d'); let scoreSpan = document.getElementById('scoreDisplay'); let epsSpan = document.getElementById('epsilonDisplay'); let episodeSpan = document.getElementById('episodeDisplay'); let speedSlider = document.getElementById('speedSlider'); let speedValue = document.getElementById('speedValue'); // snake & food let snake = []; let direction = { x: 1, y: 0 }; // initial right let nextDirection = { x: 1, y: 0 }; let food = { x: 6, y: 6 }; let score = 0; let gameOver = false; let episode = 0; let stepCount = 0; // AI control let epsilon = 1.0; let training = false; let paused = false; let lastAction = 0; // Replay memory (as arrays for simplicity) let replayMemory = []; let memoryIdx = 0; // TensorFlow models let onlineModel; let targetModel; // ---------- INITIALIZE GAME ---------- function resetGame() { snake = [ { x: 10, y: 10 }, { x: 9, y: 10 }, { x: 8, y: 10 } ]; direction = { x: 1, y: 0 }; nextDirection = { x: 1, y: 0 }; score = 0; gameOver = false; placeFood(); updateScore(); } function placeFood() { let freeCells = []; // simple: find random free cell for (let i = 0; i s.x === i && s.y === j)) { freeCells.push({ x: i, y: j }); } } } if (freeCells.length === 0) { gameOver = true; // win condition return; } const rand = Math.floor(Math.random() * freeCells.length); food = freeCells[rand]; } // ---------- GAME STEP (returns reward, done) ---------- function stepGame(action) { // action: 0=up, 1=down, 2=left, 3=right switch (action) { case 0: nextDirection = { x: 0, y: -1 }; break; case 1: nextDirection = { x: 0, y: 1 }; break; case 2: nextDirection = { x: -1, y: 0 }; break; case 3: nextDirection = { x: 1, y: 0 }; break; default: break; } // prevent reverse if ((direction.x === -nextDirection.x && direction.y === -nextDirection.y) || (direction.x === nextDirection.x && direction.y === nextDirection.y)) { // keep current direction } else { direction = { ...nextDirection }; } // compute new head const head = snake[0]; const newHead = { x: head.x + direction.x, y: head.y + direction.y }; // collision checks let reward = 0; let done = false; // wall collision if (newHead.x = GRID_SIZE || newHead.y = GRID_SIZE) { reward = -10; done = true; } // self collision (ignore tail if moving away? but we keep simple) else if (snake.some(s => s.x === newHead.x && s.y === newHead.y)) { reward = -10; done = true; } else { // move snake snake.unshift(newHead); // check food if (newHead.x === food.x && newHead.y === food.y) { score += 1; reward = 10; placeFood(); // do not remove tail -> grows } else { snake.pop(); // remove tail // small living penalty to encourage faster eating reward = 0.01; } // check if game over due to no free cells (win) if (snake.length === GRID_SIZE * GRID_SIZE) { reward = 20; done = true; } } if (done) { // extra penalty if game over if (reward === -10) reward = -10; else if (reward === 20) reward = 20; } return { reward, done }; } // ---------- STATE REPRESENTATION (11-dim) ---------- function getState() { const head = snake[0]; // danger straight, left, right const dir = direction; // points: straight, left, right const pointStraight = { x: head.x + dir.x, y: head.y + dir.y }; const pointLeft = { x: head.x - dir.y, y: head.y + dir.x }; const pointRight = { x: head.x + dir.y, y: head.y - dir.x }; const dangerStraight = (pointStraight.x = GRID_SIZE || pointStraight.y = GRID_SIZE || snake.some(s => s.x === pointStraight.x && s.y === pointStraight.y)) ? 1 : 0; const dangerLeft = (pointLeft.x = GRID_SIZE || pointLeft.y = GRID_SIZE || snake.some(s => s.x === pointLeft.x && s.y === pointLeft.y)) ? 1 : 0; const dangerRight = (pointRight.x = GRID_SIZE || pointRight.y = GRID_SIZE || snake.some(s => s.x === pointRight.x && s.y === pointRight.y)) ? 1 : 0; // direction one-hot const dirUp = dir.y === -1 ? 1 : 0; const dirDown = dir.y === 1 ? 1 : 0; const dirLeft = dir.x === -1 ? 1 : 0; const dirRight = dir.x === 1 ? 1 : 0; // food direction relative const foodUp = food.y head.y ? 1 : 0; const foodLeft = food.x head.x ? 1 : 0; return [ dangerStraight, dangerLeft, dangerRight, dirUp, dirDown, dirLeft, dirRight, foodUp, foodDown, foodLeft, foodRight ]; } // ---------- NEURAL NETWORKS (DDQN) ---------- function createModel() { const model = tf.sequential(); model.add(tf.layers.dense({ units: HIDDEN_SIZE, activation: 'relu', inputShape: [11] })); model.add(tf.layers.dense({ units: HIDDEN_SIZE, activation: 'relu' })); model.add(tf.layers.dense({ units: NUM_ACTIONS, activation: 'linear' })); model.compile({ optimizer: tf.train.adam(LEARNING_RATE), loss: 'meanSquaredError' }); return model; } // ---------- MEMORY ---------- function storeTransition(state, action, reward, nextState, done) { if (replayMemory.length < MEMORY_CAPACITY) { replayMemory.push({ state, action, reward, nextState, done }); } else { replayMemory[memoryIdx] = { state, action, reward, nextState, done }; memoryIdx = (memoryIdx + 1) % MEMORY_CAPACITY; } } // ---------- TRAINING STEP (Double DQN) ---------- async function trainBatch() { if (replayMemory.length e.state); const actions = batch.map(e => e.action); const rewards = batch.map(e => e.reward); const nextStates = batch.map(e => e.nextState); const dones = batch.map(e => e.done); const stateTensor = tf.tensor2d(states, [BATCH_SIZE, 11]); const nextStateTensor = tf.tensor2d(nextStates, [BATCH_SIZE, 11]); // Double DQN: use online to select action, target to evaluate const nextQOnline = onlineModel.predict(nextStateTensor); const nextQTarget = targetModel.predict(nextStateTensor); const nextActionIndices = tf.argMax(nextQOnline, 1).dataSync(); const nextQValues = nextQTarget.dataSync(); // compute targets const targets = []; for (let i = 0; i < BATCH_SIZE; i++) { const r = rewards[i]; const done = dones[i]; let targetQ = r; if (!done) { const maxNextQ = nextQValues[i * NUM_ACTIONS + nextActionIndices[i]]; targetQ = r + GAMMA * maxNextQ; } // current Q values const currentQ = onlineModel.predict(stateTensor).dataSync(); const qUpdate = currentQ.slice(i * NUM_ACTIONS, (i+1) * NUM_ACTIONS); qUpdate[actions[i]] = targetQ; targets.push(...qUpdate); } // train online model const targetTensor = tf.tensor2d(targets, [BATCH_SIZE, NUM_ACTIONS]); await onlineModel.fit(stateTensor, targetTensor, { epochs: 1, verbose: 0 }); // cleanup stateTensor.dispose(); nextStateTensor.dispose(); nextQOnline.dispose(); nextQTarget.dispose(); targetTensor.dispose(); } // ---------- EPSILON GREEDY ACTION ---------- function chooseAction(state) { if (Math.random() < epsilon) { return Math.floor(Math.random() * NUM_ACTIONS); } else { const stateTensor = tf.tensor2d([state], [1, 11]); const qValues = onlineModel.predict(stateTensor); const action = qValues.argMax(-1).dataSync()[0]; stateTensor.dispose(); qValues.dispose(); return action; } } // ---------- GAME LOOP (AI driven) ---------- let lastTime = 0; let frameInterval = 100; // ms per step (adjusted by speed) let animationId = null; function gameLoop(timestamp) { if (!training) { // manual mode not implemented, but we keep AI always } if (paused) { animationId = requestAnimationFrame(gameLoop); return; } const speed = parseInt(speedSlider.value); frameInterval = Math.max(20, 200 / speed); // faster speed = lower interval if (timestamp - lastTime >= frameInterval) { lastTime = timestamp; if (!gameOver) { // get state const state = getState(); // choose action const action = chooseAction(state); // step const { reward, done } = stepGame(action); const nextState = getState(); // store transition storeTransition(state, action, reward, nextState, done); stepCount++; // train if enough memory if (training && replayMemory.length >= BATCH_SIZE) { trainBatch().then(() => { // update target network periodically if (stepCount % TARGET_UPDATE_FREQ === 0) { targetModel.setWeights(onlineModel.getWeights()); } }); } // decay epsilon if (epsilon > MIN_EPSILON) { epsilon *= DECAY_RATE; epsilon = Math.max(MIN_EPSILON, epsilon); } // update UI scoreSpan.textContent = score; epsSpan.textContent = epsilon.toFixed(2); if (done) { episode++; episodeSpan.textContent = episode; resetGame(); // small extra decay epsilon = Math.max(MIN_EPSILON, epsilon * 0.99); } // draw drawCanvas(); } else { // game over -> reset episode++; episodeSpan.textContent = episode; resetGame(); } } animationId = requestAnimationFrame(gameLoop); } // ---------- DRAW ---------- function drawCanvas() { ctx.clearRect(0, 0, canvasWidth, canvasHeight); // grid ctx.strokeStyle = '#2a2a4a'; ctx.lineWidth = 0.5; for (let i = 0; i <= GRID_SIZE; i++) { ctx.beginPath(); ctx.moveTo(i * (canvasWidth/GRID_SIZE), 0); ctx.lineTo(i * (canvasWidth/GRID_SIZE), canvasHeight); ctx.stroke(); ctx.moveTo(0, i * (canvasHeight/GRID_SIZE)); ctx.lineTo(canvasWidth, i * (canvasHeight/GRID_SIZE)); ctx.stroke(); } // food ctx.fillStyle = '#ff5e5e'; ctx.shadowColor = '#ff8888'; ctx.shadowBlur = 8; ctx.beginPath(); ctx.arc(food.x * (canvasWidth/GRID_SIZE) + (canvasWidth/GRID_SIZE)/2, food.y * (canvasHeight/GRID_SIZE) + (canvasHeight/GRID_SIZE)/2, (canvasWidth/GRID_SIZE)/2 - 2, 0, 2 * Math.PI); ctx.fill(); ctx.shadowBlur = 0; // snake snake.forEach((seg, idx) => { const x = seg.x * (canvasWidth/GRID_SIZE); const y = seg.y * (canvasHeight/GRID_SIZE); const size = canvasWidth/GRID_SIZE; ctx.fillStyle = idx === 0 ? '#6bff6b' : '#3fbf3f'; ctx.shadowColor = '#88ff88'; ctx.shadowBlur = idx === 0 ? 10 : 4; ctx.fillRect(x + 1, y + 1, size - 2, size - 2); }); ctx.shadowBlur = 0; } // ---------- UI CONTROLS ---------- document.getElementById('trainBtn').addEventListener('click', () => { training = !training; document.getElementById('trainBtn').textContent = training ? '⏸️ TRAIN ON' : '▶ TRAIN'; if (training && !animationId) { lastTime = performance.now(); animationId = requestAnimationFrame(gameLoop); } }); document.getElementById('pauseBtn').addEventListener('click', () => { paused = !paused; document.getElementById('pauseBtn').textContent = paused ? '▶ RESUME' : '⏸️ PAUSE'; }); document.getElementById('resetBtn').addEventListener('click', () => { resetGame(); episode = 0; episodeSpan.textContent = episode; scoreSpan.textContent = 0; drawCanvas(); }); speedSlider.addEventListener('input', () => { speedValue.textContent = speedSlider.value + 'x'; }); // ---------- INIT ---------- async function init() { onlineModel = createModel(); targetModel = createModel(); targetModel.setWeights(onlineModel.getWeights()); resetGame(); drawCanvas(); // start loop always (for rendering) lastTime = performance.now(); animationId = requestAnimationFrame(gameLoop); // enable training by default? we start with training false, but we want AI to move // we set training = true to start learning training = true; document.getElementById('trainBtn').textContent = '⏸️ TRAIN ON'; } init();

使用TensorFlow.js构建DDQN贪吃蛇游戏

提示词内容收藏

用法

参考资料

讨论