slow runtime for large input values

Your code makes use of repeated string concatencation. Each addition operation on a string takes O(n) time, because strings are immutable. This operation occurs O(n^2) times, so the runtime of your algorithm is O(n^3), where n is the length of the string.

One optimization is to use a list to store the letters, and then call ' '.join(), taking the runtime from O(n^3) to O(n^2):

def colour_trio(colours):
    result = colours
    
    while len(result) != 1:
        letters = []
        for fst, snd in zip(result, result[1:]):
            if fst == snd:
                letters.append(fst)
            else:
                [letter_to_add] = list(set('ryb') - set(fst) - set(snd))
                letters.append(letter_to_add)
            
        result = ''.join(letters)
        
    return result
    
print(colour_trio("rybbry")) # Prints "b"

I think this is faster than the last solution I posted and faster than other solutions, I think classes make everything tidy looking, but you can re-work it to become a single function.

PS. Thank you for the feedback @KellyBundy! Now the algorithm is correct, it took even fewer chars!

New Solution

colors = ['r', 'b', 'y']

def split(x):
    return [f'{Couple(i)}' for i in [f'{x[i]}{x[i+1]}' for i in range(len(x)-1)]]

class Couple:
    def __init__(self, x) -> None:
        self.a = x[0]
        self.b = x[1]

    def __repr__(self) -> str:
        if self.a == self.b:
            return self.a
        return (set(colors).difference(set((self.a, self.b))).pop())

def ColoursTrio(x):
    while len(x) > 1:
        x = split(x)
    return x[0]

print(ColoursTrio('bbr'))

or even shorter (and faster):

def ColoursTrio(x):
    while len(x) > 1:
        x = [f'''{i[0] if i[0] == i[1] 
            else  set("rby").difference(set((i[0],
            i[1]))).pop()}''' for i in [f'{x[i]}{x[i+1]}' 
            for i in range(len(x)-1)]]
    return x[0]

print(ColoursTrio('bbr'))

Old Solution

This is almost a bit convoluted but I think it's faster, I also think it's very similar to the one posted by @BrokenBenchmark (and I think his is better) so I'll add his example string too!

COLORS = ['r','b','y']

class ColorsCouple:
    def __init__(self, x:str, y:str) -> None:
        self.x = x
        self.y = y
    
    def mix(self):
        if self.x == self.y:
            return f'{self.x}'
        if 'y' in (self.x, self.y):
            return set(COLORS).difference((self.x, self.y)).pop()
        return 'y'

class Colors:
    def __init__(self, colors:str) -> None:
        self.colors = self.get_couples([*colors])
    
    def get_couples(self, element):
        return [(element[i],element[i+1]) for i in range(len(element)-1)]

    def get_color(self):
        colors = [ColorsCouple(*couples).mix() if len(couples)==2 else couples[0] for couples in self.colors]
        while len(colors)>1:       
            colors = self.get_couples(colors)
            colors = [ColorsCouple(*couples).mix() if len(couples)==2 else couples[0] for couples in colors]
        return colors[0]

    def __repr__(self) -> str:
        return f'{self.get_color()}'

print(Colors('bbr'))
print(Colors('rybbry'))

Another solution, first some benchmark results, with random strings of 70 to 560 letters (I think 140 is the maximum in the actual tests):

 n= 70   n=140   n=280   n=560 
  0.03    0.06    0.11    0.24  Kelly_linear (I might show this one later)
  0.47    1.71    6.58   25.85  Kelly        (this is the one I am already showing)
  0.77    3.17   12.50   52.57  Jacopo_2
  1.59    6.38   25.62  107.81  Jacopo_1
  1.69    7.09   27.93  110.67  Fabio
  1.82    7.69   30.17  120.77  BrokenBenchmark

My solution is the function Kelly below. First I double every letter, then trim first and last letter. So rybbry becomes ryybbbbrry. Essentially that gives me all the pairs, if you think spaces into it then it would be ry yb bb br ry. Then I replace pairs of different letters with what that pair shall become, but in uppercase. So I get BRbbYB. Then replace pairs of equal letters with what that pair shall become: BRBYB. Then just go back to lower case. Repeat until done.

Full benchmark code (Try it online!):

def Kelly(colours):
    while len(colours) != 1:
        colours = (colours
            .replace('b', 'bb')
            .replace('r', 'rr')
            .replace('y', 'yy')
            [1:-1]
            .replace('br', 'Y').replace('rb', 'Y')
            .replace('by', 'R').replace('yb', 'R')
            .replace('ry', 'B').replace('yr', 'B')
            .replace('bb', 'B')
            .replace('rr', 'R')
            .replace('yy', 'Y')
            .lower())
    return colours

def Jacopo_1(colours):
  while len(colours) > 1:
    new_colours = ''
    for i in range(len(colours) - 1):
      if colours[i:i+2] == 'rr' or colours[i:i+2] == 'by' or colours[i:i+2] == 'yb':
        new_colours = new_colours + 'r'
      elif colours[i:i+2] == 'bb' or colours[i:i+2] == 'ry' or colours[i:i+2] == 'yr':
        new_colours = new_colours + 'b'
      else:
        new_colours = new_colours + 'y'
    colours = new_colours
  if len(colours) == 1:
    if colours[0] == 'b':
        return 'b'
    if colours[0] == 'y':
        return 'y' 
    if colours[0] == 'r':
        return 'r'

def Jacopo_2(colours):
  colours = list(colours)
  length = len(colours)
  for i in range(length-1):
    for k in range(length - 1 - i):
      colours[k] = color_sum(colours[k:k+2])
  return colours[0]
def color_sum(pair):
    if pair[0] == pair[1]:
        return pair[0]
    if 'r' in pair:
        if 'b' in pair:
            return 'y'
        return 'b'
    return 'r'

def BrokenBenchmark(colours):
    result = colours    
    while len(result) != 1:
        letters = []
        for fst, snd in zip(result, result[1:]):
            if fst == snd:
                letters.append(fst)
            else:
                [letter_to_add] = list(set('ryb') - set(fst) - set(snd))
                letters.append(letter_to_add)
            
        result = ''.join(letters)        
    return result

def Fabio(x):
    while len(x) > 1:
        x = [f'''{i[0] if i[0] == i[1] 
            else  set("rby").difference(set((i[0],
            i[1]))).pop()}''' for i in [f'{x[i]}{x[i+1]}' 
            for i in range(len(x)-1)]]
    return x[0]

lengths = 70, 140, 280, 560
funcs = [
    Kelly,
    Jacopo_2,
    Jacopo_1,
    Fabio,
    BrokenBenchmark,
]

from timeit import default_timer as timer, repeat
import random

Tss = [[] for _ in funcs]
for length in lengths:
    tss = [[] for _ in funcs]
    for _ in range(5):
        colours = ''.join(random.choices('bry', k=length))
        def run():
            global result
            result = func(colours)
        expect = None
        for func, ts in zip(funcs, tss):
            t = min(repeat(run, number=1))
            if expect is None:
                expect = result
            assert result == expect
            ts.append(t)
           # print(*('%7.3f ms ' % (t * 1e3) for t in sorted(ts)[:3]), func.__name__)
       # print()
    print(*(f' n={length:3} ' for length in lengths))
    for func, Ts, ts in zip(funcs, Tss, tss):
        Ts.append(min(ts))
        print(*('%6.2f ' % (t * 1e3) for t in Ts), func.__name__)
    print()