How run JavaScript function when a button is clicked

The first issue is the format of the color string you want to set the input as, it needs to be the right format #RRGGBB Input color value docs

let switchToRgb = `#${r.toString(16)}${g.toString(16)}${b.toString(16)}`;

See update below about properly creating this colour string

this will create the string in the right format by converting each decimal value to hexadecimal and formatting the string with a leading #.

That's right box doesn't exist at that point, you will have to get it inside the event listener. You should also set the input's value as this will change the rendered colours in the grid.

document.getElementById('bg-color').value = switchToRgb;

It strikes me as a bit weird that you're using mouseenter event on a button. Why don't you use click like most people, this will prevent accidental changes to the colour - FIXED

UPDATE - OP wants to use either a random colour each time the background color is set or use the value from color picker

You will need to keep track of whether to use a random color or use the color picked value.

let useRandomColor = false;

We'll default this to false but update it when either a specific colour is picked or the random colour button is clicked.

document.getElementById('rgb')
  .addEventListener('click', () => {
    useRandomColor = true;
  });
  
const colorPicker = document.getElementById('bg-color');
colorPicker.addEventListener('change', () => {
  useRandomColor = false;
});

Then we just need to use this value to figure out whether the picked colour is used or whether to use a random value

We also need to make sure we do this for each box

// get all the boxes 
const boxes = document.querySelectorAll('.box');
for (let box of boxes) {
  // add an event listener to each box
  box.addEventListener('mouseenter', (e) => {
    // set default value as the current value of the colour picker
    let color = colorPicker.value;

    // Override this with a random colour if needed
    if (useRandomColor){
      let r = Math.floor(Math.random() * 256);
      let g = Math.floor(Math.random() * 256);
      let b = Math.floor(Math.random() * 256);
      color = `#${r.toString(16).padStart(2, '0'}${g.toString(16)}${b.toString(16)}`;
    }

    // Set the background colour of the hovered box
    e.target.style.backgroundColor = color;
  });
};

UPDATE - Use padded hex values

As @enhzflep pointed out, the colour value needs to be exactly either 3 or 6 hex digits, so we need to ensure the generated colour has exactly 6 hex digits.

const getColourString = (r, g, b) => {
  return '#' + [r, g, b].map(x => x.toString(16).padStart(2, '0')).join('');
};

This will take our rgb colour components and make sure the value is the correct format.

We will then use this in setting the box background colour

e.target.style.backgroundColor = color;

let useRandomColor = false;

function drawGrid(container, col, row) {
  const box = document.createElement('div');
  box.className = 'box';
  box.id = `box${col}${row}`;

  container.appendChild(box);
  return (box);
}


function createBox(container) {
  const grid = document.createElement('div');
  grid.className = 'grid';

  for(let i = 0; i < 6; i++) {
    for(let j = 0; j < 5; j++) {
      drawGrid(grid, i, j);
    }
  }
  container.appendChild(grid);
}

function startupGrid() {
  const game = document.querySelector('#game');
  createBox(game)
}
startupGrid()

// When rgb button is clicked mouseenter random colors and it should ignore color pallet
document.getElementById('rgb')
  .addEventListener('click', () => {
    useRandomColor = true;
  });
  
const colorPicker = document.getElementById('bg-color');
colorPicker.addEventListener('change', () => {
  useRandomColor = false;
})

  // Wehn color pallet is clicked mouseenter radom colors and it should ignore rbg button
const boxes = document.querySelectorAll('.box');
for (let box of boxes) {
  box.addEventListener('mouseenter', (e) => {
    let color = colorPicker.value;

    if (useRandomColor){
      let r = Math.floor(Math.random() * 256);
      let g = Math.floor(Math.random() * 256);
      let b = Math.floor(Math.random() * 256);
      color = getColourString(r, g, b);
    }

    e.target.style.backgroundColor = color;
  });
};

const getColourString = (r, g, b) => {
  return '#' + [r, g, b].map(x => x.toString(16).padStart(2, '0')).join('');
};

body {
  padding: 0;
  margin: 0;
  box-sizing: border-box;
}

.main {
  display: flex;
  justify-content: center;
  align-items: center;
  place-items: center;
  gap: 2rem;
  margin: 5rem;
}

.controls {
  display: grid;
  gap: 1rem;
}

button {
  padding: 5px 15px;
}


#game {
  display: grid;
  place-items: center;
  
}

.box {
  width: 40px;
  height: 30px;
  border: 1px solid;
}

.grid {
  display: grid;
  grid-template-columns: repeat(6, auto);
  grid-template-rows: repeat(5, auto);
  box-sizing: border-box;
  gap: 1px;
  background: rgb(255, 255, 255)
}

<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width">
  <title>Grid</title>
  <link href="style.css" rel="stylesheet" type="text/css" />
</head>
<body>

  <div class="main">
    <div class="controls">
      <input type="color" id="bg-color">
      <button id="rgb">Random Colour</button>
    </div>
     
    <div id="game"></div>
  </div>
  <script src="script.js"></script>
</body>
</html>