How to retain navigation stack in Flutter when user changes system theme?

I am using ThemeExtension class to add more custom light and dark theme colors to my app. However, when I navigate to another screen from the home screen and change the theme, the app comes back to home screen.

My code is similar to this one

import 'package:flutter/material.dart';
import 'package:flutter/scheduler.dart';

/// Flutter code sample for [ThemeExtension].

@immutable
class MyColors extends ThemeExtension<MyColors> {
  const MyColors({
    required this.brandColor,
    required this.danger,
  });

  final Color? brandColor;
  final Color? danger;

  @override
  MyColors copyWith({Color? brandColor, Color? danger}) {
    return MyColors(
      brandColor: brandColor ?? this.brandColor,
      danger: danger ?? this.danger,
    );
  }

  @override
  MyColors lerp(MyColors? other, double t) {
    if (other is! MyColors) {
      return this;
    }
    return MyColors(
      brandColor: Color.lerp(brandColor, other.brandColor, t),
      danger: Color.lerp(danger, other.danger, t),
    );
  }

  // Optional
  @override
  String toString() => 'MyColors(brandColor: $brandColor, danger: $danger)';
}

void main() {
  // Slow down time to see lerping.
  timeDilation = 5.0;
  runApp(const ThemeExtensionExampleApp());
}

class ThemeExtensionExampleApp extends StatelessWidget {
  const ThemeExtensionExampleApp({super.key});

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      theme: ThemeData.light().copyWith(
        extensions: <ThemeExtension<dynamic>>[
          const MyColors(
            brandColor: Color(0xFF1E88E5),
            danger: Color(0xFFE53935),
          ),
        ],
      ),
      darkTheme: ThemeData.dark().copyWith(
        extensions: <ThemeExtension<dynamic>>[
          const MyColors(
            brandColor: Color(0xFF90CAF9),
            danger: Color(0xFFEF9A9A),
          ),
        ],
      ),
      themeMode: ThemeMode.system,
      home: Home(),
    );
  }
}

class Home extends StatelessWidget {
  const Home({super.key});

  @override
  Widget build(BuildContext context) {
    final MyColors myColors = Theme.of(context).extension<MyColors>()!;

    return Material(
      child: Center(
        child: Column(
          mainAxisAlignment: MainAxisAlignment.center,
          children: [
            Row(
              mainAxisAlignment: MainAxisAlignment.center,
              children: <Widget>[
                Container(width: 100, height: 100, color: myColors.brandColor),
                const SizedBox(width: 10),
                Container(width: 100, height: 100, color: myColors.danger),
                const SizedBox(width: 50),
              ],
            ),
            ElevatedButton(
              onPressed: () {
                Navigator.of(context).push(
                  MaterialPageRoute(builder: (_) => AnotherScreen()),
                );
              },
              child: Text('Navigate'),
            ),
          ],
        ),
      ),
    );
  }
}

class AnotherScreen extends StatelessWidget {
  const AnotherScreen({super.key});

  @override
  Widget build(BuildContext context) {
    final MyColors myColors = Theme.of(context).extension<MyColors>()!;

    return Scaffold(
      appBar: AppBar(),
      body: Center(
        child: Column(
          mainAxisAlignment: MainAxisAlignment.center,
          children: [
            Row(
              mainAxisAlignment: MainAxisAlignment.center,
              children: <Widget>[
                Container(width: 100, height: 100, color: myColors.brandColor),
                const SizedBox(width: 10),
                Container(width: 100, height: 100, color: myColors.danger),
                const SizedBox(width: 50),
              ],
            ),
            ElevatedButton(
              onPressed: () {
                Navigator.of(context).pop();
              },
              child: Text('Go back'),
            ),
          ],
        ),
      ),
    );
  }
}

import 'package:flutter/material.dart';

class MyCustomThemeExtension extends ThemeExtension<MyCustomThemeExtension> {
  final Color customColor;

  MyCustomThemeExtension({required this.customColor});

  @override
  MyCustomThemeExtension copyWith({Color? customColor}) {
    return MyCustomThemeExtension(
      customColor: customColor ?? this.customColor,
    );
  }

  @override
  MyCustomThemeExtension lerp(
      ThemeExtension<MyCustomThemeExtension>? other, double t) {
    if (other is! MyCustomThemeExtension) return this;
    return MyCustomThemeExtension(
      customColor: Color.lerp(customColor, other.customColor, t)!,
    );
  }
}

except that I am using 15-20 different custom colors, opening a sqflite database in main.

 theme: lightThemeData(context),
 darkTheme: ThemeData.dark(), //darkThemeData(context),

When I change any of the above to default constructors, navigation works as expected.