Skip to content

SinglePlayer Tournament

Overview

SinglePlayer mode has an agent play against itself.

Tournament Format

  1. Initialization: Agent starts with clean codebase
  2. Round Loop: For each round:
  3. Agent updates code
  4. Challenge/benchmark is executed
  5. Performance metrics recorded
  6. Evaluation: Agent scored on cumulative performance

Configuration

game:
  name: HuskyBench
  rounds: 10


players:
  - name: SoloAgent
    model: gpt-4-turbo
    temperature: 0.7

Running a SinglePlayer Tournament

python main_single_player.py configs/examples/battlesnake_single_player.yaml

Implementation

codeclash.tournaments.single_player.SinglePlayerTraining 

SinglePlayerTraining(config: dict, *, output_dir: Path, cleanup: bool = False, keep_containers: bool = False)

Bases: AbstractTournament

Source code in codeclash/tournaments/single_player.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
def __init__(self, config: dict, *, output_dir: Path, cleanup: bool = False, keep_containers: bool = False):
    super().__init__(config, name="SinglePlayerTraining", output_dir=output_dir)
    self.cleanup_on_end = cleanup
    self.game: CodeArena = get_game(
        self.config,
        tournament_id=self.tournament_id,
        local_output_dir=self.local_output_dir,
        keep_containers=keep_containers,
    )
    self.agent: Player = self.get_agent(self.config["player"], round=1)
    mirror_agent_config = copy.deepcopy(self.config["player"])
    mirror_agent_config["name"] = "mirror"
    self.mirror_agent: Player = self.get_agent(mirror_agent_config, round=0)

cleanup_on_end instance-attribute 

cleanup_on_end = cleanup

game instance-attribute 

game: CodeArena = get_game(config, tournament_id=tournament_id, local_output_dir=local_output_dir, keep_containers=keep_containers)

agent instance-attribute 

agent: Player = get_agent(config['player'], round=1)

mirror_agent instance-attribute 

mirror_agent: Player = get_agent(mirror_agent_config, round=0)

rounds property 

rounds: int

get_metadata 

get_metadata() -> dict
Source code in codeclash/tournaments/single_player.py 
42
43
44
45
46
47
def get_metadata(self) -> dict:
    return {
        **super().get_metadata(),
        "game": self.game.get_metadata(),
        "agents": [self.agent.get_metadata(), self.mirror_agent.get_metadata()],
    }

get_game_context 

get_game_context(agent_config: dict, *, round: int) -> GameContext

Create a game context for an agent.

Source code in codeclash/tournaments/single_player.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
def get_game_context(self, agent_config: dict, *, round: int) -> GameContext:
    """Create a game context for an agent."""
    return GameContext(
        id=self.game.game_id,
        log_env=self.game.log_env,
        log_local=self.game.log_local,
        name=self.game.name,
        player_id=agent_config["name"],
        prompts=self.config["prompts"],
        round=round,
        rounds=self.rounds,
        working_dir=str(DIR_WORK),
    )

get_agent 

get_agent(agent_config: dict, round: int) -> Player

Create an agent with environment and game context.

Source code in codeclash/tournaments/single_player.py 
63
64
65
66
67
def get_agent(self, agent_config: dict, round: int) -> Player:
    """Create an agent with environment and game context."""
    environment = self.game.get_environment(f"{self.game.game_id}.{agent_config['name']}")
    game_context = self.get_game_context(agent_config, round=round)
    return get_agent(agent_config, game_context, environment)

get_dummy_agent 

get_dummy_agent(player_config: dict) -> Player

Create a dummy agent that does nothing.

Source code in codeclash/tournaments/single_player.py 
77
78
79
80
81
82
83
def get_dummy_agent(self, player_config: dict) -> Player:
    """Create a dummy agent that does nothing."""
    return Dummy(
        player_config,
        environment=self.game.get_environment(f"{self.game.game_id}.dummy"),
        game_context=self.get_game_context(player_config, round=0),
    )

run 

run()

Main execution function that runs all rounds.

Source code in codeclash/tournaments/single_player.py 
85
86
87
88
89
90
91
92
93
def run(self):
    """Main execution function that runs all rounds."""
    try:
        for round_num in range(1, self.rounds + 1):
            self.run_training_round(round_num)
        if self.config["tournament"]["evaluate_matrix"]:
            self.evaluate()
    finally:
        self.end()

run_training_round 

run_training_round(round_num: int) -> None

Execute a single training round, i.e., run the game, then run the agent.

Source code in codeclash/tournaments/single_player.py 
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def run_training_round(self, round_num: int) -> None:
    """Execute a single training round, i.e., run the game, then run the agent."""
    # Run the game round and get results
    stats = self.game.run_round([self.agent, self.mirror_agent], round_num)
    self.logger.info(stats)
    self._metadata.setdefault("round_stats", {})[round_num] = stats.to_dict()

    # Write log to file
    results_file = self.game.log_local / "rounds" / str(round_num) / FILE_RESULTS
    results_file.write_text(json.dumps(stats.to_dict(), indent=2))

    # Copy log to main agent environment only
    self.logger.info(f"Copying round {round_num} log(s) to {self.agent.name}'s container...")
    copy_to_container(
        self.agent.environment,
        self.game.log_local / "rounds" / str(round_num),
        f"logs/rounds/{round_num}/",
    )

    self.run_main_agent(round_num)
    mirror_agent_state = round_num - 1 if round_num > 1 else 0
    self.set_mirror_state_to_round(mirror_agent_state)

    self._save()

    self.logger.info("Round completed.")

run_main_agent 

run_main_agent(round_num: int)

Run the main agent for the current round.

Source code in codeclash/tournaments/single_player.py 
122
123
124
125
126
def run_main_agent(self, round_num: int):
    """Run the main agent for the current round."""
    self.agent.pre_run_hook(new_round=round_num)
    self.agent.run()
    self.agent.post_run_hook(round=round_num)

set_mirror_state_to_round 

set_mirror_state_to_round(round_num: int)

Update mirror agent's codebase with the main agent's changes.

Source code in codeclash/tournaments/single_player.py 
128
129
130
131
132
def set_mirror_state_to_round(self, round_num: int):
    """Update mirror agent's codebase with the main agent's changes."""
    full_diff = self._get_round_diff(self.agent.name, round_num)
    full_diff = filter_git_diff(full_diff)
    self.mirror_agent.reset_and_apply_patch(full_diff)

end 

end()

Clean up game resources.

Source code in codeclash/tournaments/single_player.py 
140
141
142
143
def end(self):
    """Clean up game resources."""
    self._save()
    self.game.end(self.cleanup_on_end)

evaluate 

evaluate(n_repetitions: int = 3) -> None

Evaluate the agent's performance by calculating the matrix of every round against each other.

Source code in codeclash/tournaments/single_player.py 
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
def evaluate(self, n_repetitions: int = 3) -> None:
    """Evaluate the agent's performance by
    calculating the matrix of every round against each other.
    """
    p1_config = self.config["player"].copy()
    p1_config["name"] = "p1"
    p1 = self.get_dummy_agent(p1_config)

    p2_config = self.config["player"].copy()
    p2_config["name"] = "p2"
    p2 = self.get_dummy_agent(p2_config)
    matrix = {
        p1_round: {p2_round: [] for p2_round in range(0, self.rounds + 1)} for p1_round in range(0, self.rounds + 1)
    }
    for p1_round in range(0, self.rounds + 1):
        for p2_round in range(0, self.rounds + 1):
            self.logger.info(f"Evaluating agent at round {p1_round} against agent at round {p2_round}")
            p1_patch = self._get_round_diff(self.agent.name, p1_round)
            p2_patch = self._get_round_diff(self.agent.name, p2_round)
            p1.reset_and_apply_patch(p1_patch)
            p2.reset_and_apply_patch(p2_patch)
            for i_repetition in range(n_repetitions):
                stats = self.game.run_round([p1, p2], round_num=int(f"{p1_round}{p2_round}{i_repetition}"))
                self.logger.info(f"Round {p1_round} vs {p2_round} repetition {i_repetition} winner: {stats.winner}")
                matrix[p1_round][p2_round].append(stats.winner)
    self.logger.info(f"Evaluation matrix: {matrix}")
    self._metadata.setdefault("evaluation", {})["matrix"] = matrix
bug_report Something broken/unclear?

Open an issue on GitHub!
help Open-ended discussions

Join our Slack!
Our projects
SWE-bench SWE-bench SWE-agent SWE-agent SWE-smith SWE-smith SWE-rex SWE-ReX sb-cli sb-cli Slack Join our Slack