import numpy as np
import matplotlib.pyplot as plt

class MultiArmedBandit:
    def __init__(self, n_arms=10):
        self.true_rewards = np.random.normal(0, 1, n_arms)
        self.n_arms = n_arms
        print("每臺(tái)老虎機(jī)的真實(shí)獎(jiǎng)勵(lì)值（智能體未知）:", self.true_rewards)
        print("最優(yōu)的老虎機(jī)是 #", np.argmax(self.true_rewards), "，預(yù)期獎(jiǎng)勵(lì)為",
              np.max(self.true_rewards))

    def pull(self, arm):
        return np.random.normal(self.true_rewards[arm], 1)


class EpsilonGreedyAgent:
    def __init__(self, n_arms=10, epsilon=0.1, learning_rate=0.1):
        self.n_arms = n_arms
        self.epsilon = epsilon
        self.learning_rate = learning_rate
        self.q_values = np.zeros(n_arms)
        self.arm_counts = np.zeros(n_arms)

    def select_action(self):
        if np.random.random() < self.epsilon:
            return np.random.randint(self.n_arms)
        else:
            return np.argmax(self.q_values)

    def update(self, arm, reward):
        self.arm_counts[arm] += 1
        self.q_values[arm] += self.learning_rate * (reward - self.q_values[arm])


def run_bandit_experiment(n_arms=10, n_steps=1000, epsilon=0.1, learning_rate=0.1, random_seed=42):
    np.random.seed(random_seed)
    bandit = MultiArmedBandit(n_arms)
    agent = EpsilonGreedyAgent(n_arms, epsilon=epsilon, learning_rate=learning_rate)
    rewards = np.zeros(n_steps)
    optimal_actions = np.zeros(n_steps)
    optimal_arm = np.argmax(bandit.true_rewards)
    for step in range(n_steps):
        arm = agent.select_action()
        optimal_actions[step] = 1 if arm == optimal_arm else 0
        reward = bandit.pull(arm)
        rewards[step] = reward
        agent.update(arm, reward)
    cumulative_average_reward = np.cumsum(rewards) / (np.arange(n_steps) + 1)
    optimal_action_percentage = np.cumsum(optimal_actions) / (np.arange(n_steps) + 1)
    return {
       'rewards': rewards,
        'cumulative_average_reward': cumulative_average_reward,
        'optimal_action_percentage': optimal_action_percentage,
        'agent': agent,
        'bandit': bandit,
        'optimal_arm': optimal_arm
    }


def visualize_results(results):
    plt.figure(figsize=(12, 10))
    plt.subplot(2, 1, 1)
    plt.plot(results['cumulative_average_reward'])
    plt.xlabel('步驟')
    plt.ylabel('平均獎(jiǎng)勵(lì)')
    plt.title('隨時(shí)間的平均獎(jiǎng)勵(lì)')
    plt.grid(True, alpha=0.3)
    plt.subplot(2, 1, 2)
    plt.plot(results['optimal_action_percentage'])
    plt.xlabel('步驟')
    plt.ylabel('最優(yōu)行動(dòng)百分比')
    plt.title('選擇最優(yōu)行動(dòng)的頻率')
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()
    print("\n最終結(jié)果:")
    print(f"智能體對(duì)每臺(tái)老虎機(jī)的最終價(jià)值估計(jì): {results['agent'].q_values.round(3)}")
    print(f"真實(shí)獎(jiǎng)勵(lì)值: {results['bandit'].true_rewards.round(3)}")
    print(f"最優(yōu)老虎機(jī): {results['optimal_arm']}")
    print(f"每臺(tái)老虎機(jī)被拉動(dòng)的次數(shù): {results['agent'].arm_counts.astype(int)}")
    plt.figure(figsize=(10, 5))
    plt.bar(range(len(results['agent'].arm_counts)), results['agent'].arm_counts)
    plt.xlabel('老虎機(jī)')
    plt.ylabel('拉動(dòng)次數(shù)')
    plt.title('老虎機(jī)選擇分布')
    plt.show()


results = run_bandit_experiment(n_arms=10, n_steps=1000, epsilon=0.1)
visualize_results(results)


def compare_epsilons():
    epsilons = [0.01, 0.1, 0.5]
    plt.figure(figsize=(15, 6))
    for i, epsilon in enumerate(epsilons):
        results = run_bandit_experiment(epsilon=epsilon, random_seed=42)
        plt.subplot(1, 2, 1)
        plt.plot(results['cumulative_average_reward'], label=f'ε={epsilon}')
        plt.subplot(1, 2, 2)
        plt.plot(results['optimal_action_percentage'], label=f'ε={epsilon}')
        plt.subplot(1, 2, 1)
        plt.xlabel('步驟')
        plt.ylabel('平均獎(jiǎng)勵(lì)')
        plt.title('隨時(shí)間的平均獎(jiǎng)勵(lì)')
        plt.legend()
        plt.grid(True, alpha=0.3)
        plt.subplot(1, 2, 2)
        plt.xlabel('步驟')
        plt.ylabel('最優(yōu)行動(dòng)百分比')
        plt.title('最優(yōu)行動(dòng)頻率')
        plt.legend()
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.show()

class MultiArmedBandit:
    def __init__(self, n_arms=10):
        self.true_rewards = np.random.normal(0, 1, n_arms)
        self.n_arms = n_arms

def pull(self, arm):
    return np.random.normal(self.true_rewards[arm], 1)

class EpsilonGreedyAgent:
    def __init__(self, n_arms=10, epsilnotallow=0.1, learning_rate=0.1):
        self.epsilon = epsilon
        self.q_values = np.zeros(n_arms)

def select_action(self):
    if np.random.random() < self.epsilon:
        return np.random.randint(self.n_arms)
    else:
        return np.argmax(self.q_values)

def update(self, arm, reward):
    self.arm_counts[arm] += 1
    self.q_values[arm] += self.learning_rate * (reward - self.q_values[arm])

for step in range(n_steps):
    arm = agent.select_action()
    reward = bandit.pull(arm)
    agent.update(arm, reward)