Angularjs: Allow only numbers to be typed into tel and numeric input

I am trying to prevent non-numeric characters from ever showing in tel and numeric inputs with an angularjs directive.

<input type="tel" ... />
<input type="number" ... />

I found this almost perfect Stack Overflow answer, however it only works with text inputs. I need to use tel and number inputs so the correct keyboard appears on mobile devices.

How can this directive be made to work with tel AND number inputs?

var app = angular.module('myApp', []);

app.controller('MainCtrl', function($scope) {
});

app.directive('validNumber', function() {
  return {
    require: '?ngModel',
    link: function(scope, element, attrs, ngModelCtrl) {
      if(!ngModelCtrl) {
        return; 
      }

      ngModelCtrl.$parsers.push(function(val) {
        if (angular.isUndefined(val)) {
            var val = '';
        }
        var clean = val.replace( /[^0-9]+/g, '');
        if (val !== clean) {
          ngModelCtrl.$setViewValue(clean);
          ngModelCtrl.$render();
        }
        return clean;
      });

      element.bind('keypress', function(event) {
        if(event.keyCode === 32) {
          event.preventDefault();
        }
      });
    }
  };
});

Codepen using the almost-working directive can be found here.