package aima.core.probability.mdp;

import org.junit.Before;
import org.junit.Test;

import aima.core.environment.gridworld.GridCell;
import aima.core.environment.gridworld.GridWorld;
import aima.core.environment.gridworld.GridWorldAction;
import aima.core.environment.gridworld.GridWorldFactory;
import aima.core.probability.example.MDPFactory;
import aima.core.probability.mdp.MarkovDecisionProcess;
import aima.core.probability.mdp.impl.ModifiedPolicyEvaluation;
import aima.core.probability.mdp.search.PolicyIteration;

/**
 * @author Ravi Mohan
 * @author Ciaran O'Reilly
 * 
 */
public class PolicyIterationTest {
	public static final double DELTA_THRESHOLD = 1e-3;

	private GridWorld<Double> gw = null;
	private MarkovDecisionProcess<GridCell<Double>, GridWorldAction> mdp = null;
	private PolicyIteration<GridCell<Double>, GridWorldAction> pi = null;

	final int maxTiles = 6;
	final int maxScore = 10;

	@Before
	public void setUp() {
		// take 10 turns to place 6 tiles
		double defaultPenalty = -0.04;

		gw = GridWorldFactory.createGridWorldForTileGame(maxTiles, maxScore,
				defaultPenalty);
		mdp = MDPFactory.createMDPForTileGame(gw, maxTiles, maxScore);

		// gamma = 1.0
		PolicyEvaluation<GridCell<Double>, GridWorldAction> pe = new ModifiedPolicyEvaluation<GridCell<Double>, GridWorldAction>(
				100, 0.9);
		pi = new PolicyIteration<GridCell<Double>, GridWorldAction>(pe);
	}

	@Test
	public void testPolicyIterationForTileGame() {
		Policy<GridCell<Double>, GridWorldAction> policy = pi
				.policyIteration(mdp);

		for (int j = maxScore; j >= 1; j--) {
			StringBuilder sb = new StringBuilder();
			for (int i = 1; i <= maxTiles; i++) {
				sb.append(policy.action(gw.getCellAt(i, j)));
				sb.append(" ");
			}
			System.out.println(sb.toString());
		}
	}
}