Bot: adaptive bot

Avg score
6.624
Wins
3306
Losses
8815
Win rate
23.4%
Score history

Avg: 6.624
Recent battles

Source

const ALPHA = 0.16
const GAMMA = 0.88
const INITIAL_C = 2.95
const INITIAL_D = 1.15
const MIN_EXPLORE = 0.035
const MAX_EXPLORE = 0.22

function freshMemory() {
  return {
    q: {},
    visits: {},
    lastState: null,
    lastAction: null,
    lastReward: null,
    round: 0,
    totalReward: 0,
    coopCount: 0,
    defectCount: 0,
    mutualD: 0,
    mutualC: 0
  }
}

function reward(you, opponent) {
  if (you === "C" && opponent === "C") return 3
  if (you === "C" && opponent === "D") return 0
  if (you === "D" && opponent === "C") return 5
  return 1
}

function outcome(you, opponent) {
  return you + opponent
}

function rate(history, move, window = history.length) {
  const h = history.slice(-window)

  if (h.length === 0) return 0

  let count = 0

  for (const r of h) {
    if (r.opponent === move) count++
  }

  return count / h.length
}

function responseRate(history, ourMove, theirMove, window = history.length) {
  const h = history.slice(-window)

  let total = 0
  let matches = 0

  for (const r of h) {
    if (r.you === ourMove) {
      total++

      if (r.opponent === theirMove) {
        matches++
      }
    }
  }

  if (total === 0) return 0.5

  return matches / total
}

function bucket(value) {
  if (value < 0.2) return "0"
  if (value < 0.4) return "1"
  if (value < 0.6) return "2"
  if (value < 0.8) return "3"
  return "4"
}

function makeState(history) {
  if (history.length === 0) {
    return "START"
  }

  const recent = history.slice(-3)

  let pattern = ""

  for (const r of recent) {
    pattern += outcome(r.you, r.opponent)
  }

  while (pattern.length < 6) {
    pattern = "X" + pattern
  }

  const coop = bucket(rate(history, "C", 8))
  const punishment = bucket(
    responseRate(history, "C", "D", 8)
  )
  const forgiveness = bucket(
    responseRate(history, "D", "C", 8)
  )

  const last = history.at(-1)

  let mutual = "N"

  if (last.you === "D" && last.opponent === "D") {
    mutual = "D"
  } else if (last.you === "C" && last.opponent === "C") {
    mutual = "C"
  }

  return `${pattern}|${coop}|${punishment}|${forgiveness}|${mutual}`
}

function ensureState(memory, state) {
  if (!memory.q[state]) {
    memory.q[state] = {
      C: INITIAL_C,
      D: INITIAL_D
    }
  }

  if (!memory.visits[state]) {
    memory.visits[state] = {
      C: 0,
      D: 0
    }
  }
}

function bestValue(memory, state) {
  ensureState(memory, state)

  return Math.max(
    memory.q[state].C,
    memory.q[state].D
  )
}

function exploration(memory, state) {
  ensureState(memory, state)

  const visits =
    memory.visits[state].C +
    memory.visits[state].D

  if (visits === 0) {
    return MAX_EXPLORE
  }

  return Math.max(
    MIN_EXPLORE,
    MAX_EXPLORE / Math.sqrt(visits + 1)
  )
}

function chooseExploration(memory, state) {
  ensureState(memory, state)

  const c = memory.visits[state].C
  const d = memory.visits[state].D

  if (c === 0 && d > 2) return "C"
  if (d === 0 && c > 2) return "D"

  const chance = exploration(memory, state)

  const value = Math.sin(
    memory.round * 12.9898 +
    state.length * 78.233 +
    c * 37.719 +
    d * 11.131
  )

  const normalized = Math.abs(value % 1)

  if (normalized < chance) {
    if (memory.q[state].C === memory.q[state].D) {
      return memory.round % 2 === 0 ? "C" : "D"
    }

    return memory.q[state].C < memory.q[state].D
      ? "C"
      : "D"
  }

  if (memory.q[state].C >= memory.q[state].D) {
    return "C"
  }

  return "D"
}

function updateQ(memory, nextState, rewardValue) {
  if (!memory.lastState || !memory.lastAction) {
    return
  }

  ensureState(memory, memory.lastState)
  ensureState(memory, nextState)

  const oldValue =
    memory.q[memory.lastState][memory.lastAction]

  const future = bestValue(memory, nextState)

  const target =
    rewardValue +
    GAMMA * future

  memory.q[memory.lastState][memory.lastAction] =
    oldValue +
    ALPHA * (target - oldValue)

  memory.visits[memory.lastState][memory.lastAction]++
}

function updateStats(memory, history) {
  if (history.length === 0) return

  const last = history.at(-1)

  memory.lastReward =
    reward(last.you, last.opponent)

  memory.totalReward += memory.lastReward

  if (last.opponent === "C") {
    memory.coopCount++
  } else {
    memory.defectCount++
  }

  if (last.you === "C" && last.opponent === "C") {
    memory.mutualC++
    memory.mutualD = 0
  } else if (
    last.you === "D" &&
    last.opponent === "D"
  ) {
    memory.mutualD++
    memory.mutualC = 0
  } else {
    memory.mutualC = 0
    memory.mutualD = 0
  }
}

function safetyOverride(history, memory, learnedMove) {
  if (history.length < 3) {
    return "C"
  }

  const last = history.at(-1)

  const coopRate = rate(history, "C", 6)
  const defectRate = rate(history, "D", 6)

  if (memory.mutualD >= 2) {
    return "C"
  }

  if (
    defectRate >= 0.83 &&
    coopRate <= 0.17
  ) {
    return "D"
  }

  if (
    memory.mutualC >= 4 &&
    coopRate >= 0.9
  ) {
    const dAfterD =
      responseRate(history, "D", "C", 8)

    if (dAfterD >= 0.75) {
      return "D"
    }
  }

  if (
    last.opponent === "D" &&
    coopRate >= 0.7
  ) {
    return "C"
  }

  return learnedMove
}

function choose(memory, history, state) {
  ensureState(memory, state)

  let move = chooseExploration(memory, state)

  move = safetyOverride(
    history,
    memory,
    move
  )

  return move
}

export default function bot({ history, memory }) {
  if (!memory) {
    memory = freshMemory()
  }

  memory.round = history.length

  if (history.length > 0) {
    updateStats(memory, history)

    const currentState =
      makeState(history)

    updateQ(
      memory,
      currentState,
      memory.lastReward
    )
  }

  if (history.length === 0) {
    memory.lastState = "START"
    memory.lastAction = "C"
    memory.visits.START = {
      C: 1,
      D: 0
    }

    return ["C", memory]
  }

  const state = makeState(history)

  const move =
    choose(memory, history, state)

  memory.lastState = state
  memory.lastAction = move

  return [move, memory]
}