Dynamically remove entity_reference value from views exposed filter via flag

<?php

use Drupal\Component\Utility\Html;
use Drupal\Core\Form\FormStateInterface;


/**
 * Implements hook_form_FORM_ID_alter() for 'views_exposed_form'.
 */
function hook_form_views_exposed_form_alter(&$form, FormStateInterface $form_state) {
  // Configure the values to your needs.
  $view_name = 'my_view';
  $display_name = 'my_display';
  $entity_type_id = 'taxonomy_term';
  $reference_field = 'field_category';
  // The field on the referenced entity.
  $flag_on_reference_field = 'field_hide_on_exposed_form';

  if ($form['#id'] == Html::getId("views-exposed-form-{$view_name}-{$display_name}")) {
    $entity_storage = \Drupal::entityTypeManager()->getStorage($entity_type_id);
    $options = &$form[$reference_field]['#options'];
    foreach ($entity_storage->loadMultiple(array_keys($options)) as $id => $entity) {
      if ($entity->hasField($flag_on_reference_field) && !$entity->{$flag_on_reference_field}->isEmpty()) {
        unset($options[$id]);
      }
    }
  }
}