Реализация Q-обучения на Python

Определение среды

import numpy as np

# Define the environment
num_states = 16  # 4x4 grid
num_actions = 4  # Up, Down, Left, Right

# Define rewards (-1 for each step, +10 for the goal)
rewards = np.zeros(num_states)
rewards[-1] = 10

Инициализация Q-таблицы

# Initialize the Q-Table
q_table = np.zeros((num_states, num_actions))

print("Initial Q-Table:")
print(q_table)

Реализация алгоритма Q-обучения

# Parameters
alpha = 0.1  # Learning rate
gamma = 0.9  # Discount factor
epsilon = 0.1  # Exploration rate

# Q-Learning loop
num_episodes = 1000
for episode in range(num_episodes):
    state = 0  # Start at the first state
    done = False
    while not done:
        # Exploration or exploitation
        if np.random.rand() < epsilon:
            action = np.random.randint(num_actions)  # Explore
        else:
            action = np.argmax(q_table[state])  # Exploit

        # Transition to the next state
        next_state = (state + action) % num_states  # Simplified transition logic
        reward = rewards[next_state]

        # Update Q-value
        q_table[state, action] = q_table[state, action] + alpha * (
            reward + gamma * np.max(q_table[next_state]) - q_table[state, action]
        )

        # Move to the next state
        state = next_state
        done = state == num_states - 1

Оценка обученной политики

# Evaluate the policy
state = 0
optimal_path = [state]
while state != num_states - 1:
    action = np.argmax(q_table[state])
    state = (state + action) % num_states  # Simplified transition logic
    optimal_path.append(state)

print("Optimal Path:", optimal_path)

Визуализация Q-таблицы

import matplotlib.pyplot as plt
import seaborn as sns

# Visualize Q-Table
plt.figure(figsize=(10, 8))
sns.heatmap(q_table, annot=True, fmt=".2f", cmap="coolwarm")
plt.title("Q-Table Heatmap")
plt.xlabel("Actions")
plt.ylabel("States")
plt.show()

Преимущества Q-обучения

Ограничения Q-обучения

Итоги и дальнейшие шаги