Symfony 2 custom authentication not using correct provider

I'm using FOSRestBundle to create a REST API. For authentication, I'm using a header, which is sent with every request. It's very similar to this cookbook entry.

The listener works fine. Once it calls the following line, I don't see any of my debug or error log entries, it just throws an AuthenticationError exception: $returnValue = $this->authenticationManager->authenticate($token);

I suspect the provider being called is main and not the one I added named api.

security.yml is the only config file that really has much deviation from the cookbook entry:

security:
    encoders:
        Keobi\ModelBundle\Entity\User:
            algorithm: sha512
            iterations: 5000
            encode_as_base64: true

    role_hierarchy:
        ROLE_ADMIN:       [ROLE_USER, ROLE_ALLOWED_TO_SWITCH]
        ROLE_SUPER_ADMIN: [ROLE_USER, ROLE_ADMIN, ROLE_ALLOWED_TO_SWITCH]

    providers:
        main:
            entity: { class: KeobiModelBundle:User, property: email }
        api:
            entity: { class: KeobiModelBundle:Api, property: key }

    factories:
        - "%kernel.root_dir%/../src/Keobi/SecurityBundle/Resources/config/secrity_factories.yml"

    firewalls:
        dev:
            pattern:  ^/(_(profiler|wdt)|css|images|js)/
            security: false

        login:
            pattern:  ^/security/login$
            security: false

        api: # <- this is the firewall for my custom auth
            pattern: ^/api/
            #security: false
            api: true
            provider: api

        secured_area:
            pattern:    ^/(keobi|customer|security)/.*$
            form_login:
                check_path: /security/login_check
                login_path: /security/login
                success_handler: keobi_security.handler.authentication
                failure_handler: keobi_security.handler.authentication
                default_target_path: /
                target_path_parameter: _target_path
            logout:
                path:   /security/logout
                target: /security/login
                handlers: [keobi_security.handler.authentication]
            switch_user: { role: ROLE_ADMIN }

    access_control:
        - { path: ^/login, roles: IS_AUTHENTICATED_ANONYMOUSLY }
        - { path: ^/, roles: ROLE_ADMIN }

Here is my ApiListener.php file:

<?php
namespace Keobi\SecurityBundle\Listener;

use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpKernel\Event\GetResponseEvent;
use Symfony\Component\Security\Http\Firewall\ListenerInterface;
use Symfony\Component\Security\Core\Exception\AuthenticationException;
use Symfony\Component\Security\Core\SecurityContextInterface;
use Symfony\Component\Security\Core\Authentication\AuthenticationManagerInterface;
use Symfony\Component\Security\Core\Authentication\Token\TokenInterface;
use Keobi\SecurityBundle\Token\ApiToken;
use Symfony\Bridge\Monolog\Logger;

class ApiListener implements ListenerInterface
{
  protected $securityContext;
  protected $authenticationManager;
  protected $logger;
  protected $kernel;

  const AUTH_HEADER = 'x-keobi-authenticate';
  const AUTH_PATTERN = '/^Key="(?P<key>\w{40})", Hash="(?P<hash>\w+)", Created="(?P<created>\d+)"$/';
  const SIGN_HEADER = 'x-keobi-signature';

  public function __construct(SecurityContextInterface $securityContext, AuthenticationManagerInterface $authenticationManager, Logger $logger, \AppKernel $kernel)
  {
    $this->securityContext = $securityContext;
    $this->authenticationManager = $authenticationManager;
    $this->logger = $logger;
    $this->kernel = $kernel;
  }

  public function handle(GetResponseEvent $event)
  {
    $request = $event->getRequest();
    $kernel = $event->getKernel();

    if ($this->kernel->isDebug() && $request->query->has('_apikey') && $request->query->has('_apisecret') && $request->query->has('_ipaddress'))
    {
      $this->logger->debug('Debug key and secret used.');
      $token = new ApiToken();

      $created = time();
      $hash = hash('sha256', $request->query->get('_apikey') . $request->query->get('_apisecret') . strval($created));

      $token->key = $request->query->get('_apikey');
      $token->created = $created;
      $token->hash = $hash;
      $token->ipaddress = $request->query->get('_ipaddress');
    }
    elseif ($request->headers->has(self::AUTH_HEADER))
    {
      if (preg_match(self::AUTH_PATTERN, $request->headers->get(self::AUTH_HEADER), $matches))
      {
        $token = new ApiToken();

        $token->key = $matches['key'];
        $token->created = $matches['created'];
        $token->hash = $matches['hash'];
        $token->ipaddress = $request->getClientIp();
      }
    }

    if (isset($token))
    {
      $this->logger->debug($request->headers->get(self::AUTH_HEADER));

      try
      {
        $this->logger->debug(get_class($this->authenticationManager));
        $returnValue = $this->authenticationManager->authenticate($token);

        if ($returnValue instanceof TokenInterface)
          return $this->securityContext->setToken($returnValue);
        elseif ($returnValue instanceof Response)
          return $event->setResponse($returnValue);
      }
      catch (AuthenticationException $e)
      {
        $this->logger->err('Server failed to authenticate');
      }
    }

    # could not authenticate
    $response = new Response();
    $response->setStatusCode(403);
    $response->setContent('Could not be authenticated.');
    $event->setResponse($response);
  }
}

Since I posted the listener and the listener is what is generating log entries, these are the log entries that happen when attempting authentication:

2012-07-07 21:47:17 [2fiespfh-4b5a19dd] app.DEBUG: Key="0123456789012345678901234567890123456789", Hash="05707425769f01a82e2eee0b85018feeb6b96579f376f4632782b6b61c83b1fe", Created="1341655731"
2012-07-07 21:47:17 [2fiespfh-4b5a19dd] app.DEBUG: Symfony\Component\Security\Core\Authentication\AuthenticationProviderManager
2012-07-07 21:47:17 [2fiespfh-4b5a19dd] app.ERROR: Server failed to authenticate