how to add custom button inside html player using mediaelement,js script

You need to do it as follows (this is an example for a Loop button):

MediaElementPlayer.prototype.buildloop = function(player, controls, layers, media) {
    var
        // create the loop button
        loop =
        $('<div class="mejs-button mejs-loop-button ' + ((player.options.loop) ? 'mejs-loop-on' : 'mejs-loop-off') + '">' +
            '<span></span>' +
        '</div>')
        // append it to the toolbar
        .appendTo(controls)
        // add a click toggle event
        .click(function() {
            player.options.loop = !player.options.loop;
            if (player.options.loop) {
                loop.removeClass('mejs-loop-off').addClass('mejs-loop-on');
            } else {
                loop.removeClass('mejs-loop-on').addClass('mejs-loop-off');
            }
        });
}

Then, when creating your video player you can just add your variable to the features list for example:

$('video,audio').mediaelementplayer({
    features: ['loop','playpause','current','progress','duration','fullscreen'],
    alwaysShowControls: true,
});

Thank you @Sam, i used your code and wrote a vanilla version of your solution. This one adds two buttons to adjust the volume, a plus and a minus button, to make 10 steps for adjustment. (mediaelementjs 4.2.8)

Javascript:

var audio_player = document.getElementById('audio-player').children[0];

MediaElementPlayer.prototype.buildvolume_plus = function(player, controls) {
  var
    volume = document.createElement('div'),
    volume_button = document.createElement('button');
  volume.className = 'mejs__button mejs__volumeplus-button';
  volume_button.type = 'button';
  volume.appendChild(volume_button);

  controls.appendChild(volume);

  volume_button.addEventListener('click', function() {
    player.setVolume( player.volume < 1 ? Math.round( (player.volume + .1 )  * 10) / 10 : 1 );

  })
};
MediaElementPlayer.prototype.buildvolume_minus = function(player, controls) {
  var
    volume = document.createElement('div'),
    volume_button = document.createElement('button');
  volume.className = 'mejs__button mejs__volumeminus-button';
  volume_button.type = 'button';
  volume.appendChild(volume_button);

  controls.appendChild(volume);

  volume_button.addEventListener('click', function() {
    player.setVolume( player.volume > 0 ? Math.round( (player.volume - .1 )  * 10) / 10 : 0 );
  })
};

new MediaElementPlayer(audio_player);

HTML:

<div id="audio-player">
  <audio src="http://example.com" width="220" height="60" controls data-mejsoptions=\'{"features": ["playpause", "volume_plus", "volume_minus"]}\'></audio>
</div>