How can I implement/Create role based custom user authentication in asp.net MVC 3

You could use a custom authorize attribute and store the roles in the user data part of the authentication cookie. So for example inside your LogOn method once you have verified the credentials you could retrieve the roles for the given user from your database and store them into the user data:

// TODO: fetch roles from your database based on the username
var roles = "Admin|SomeRole";
var ticket = new FormsAuthenticationTicket(
    1, 
    username,
    DateTime.Now, 
    DateTime.Now.AddMilliseconds(FormsAuthentication.Timeout.TotalMilliseconds), 
    false, 
    roles
);
var encryptedTicket = FormsAuthentication.Encrypt(ticket);
var authCookie = new HttpCookie(FormsAuthentication.FormsCookieName, encryptedTicket)
{
    Domain = FormsAuthentication.CookieDomain,
    HttpOnly = true,
    Secure = FormsAuthentication.RequireSSL,
};

// Emit the authentication cookie which in addition to the username will
// contain the roles in the user data part
Response.AppendCookie(authCookie);

Then you could write a custom authorize attribute which will be used to read the authentication cookie and extract the roles information:

public class MyAuthorizeAttribute : AuthorizeAttribute
{
    protected override bool AuthorizeCore(HttpContextBase httpContext)
    {
        if (httpContext.User.Identity.IsAuthenticated)
        {
            var authCookie = httpContext.Request.Cookies[FormsAuthentication.FormsCookieName];
            if (authCookie != null)
            {
                var ticket = FormsAuthentication.Decrypt(authCookie.Value);
                var identity = new GenericIdentity(httpContext.User.Identity.Name);
                var roles = (ticket.UserData ?? string.Empty).Split('|');
                httpContext.User = new GenericPrincipal(identity, roles);
            }
        }

        return base.AuthorizeCore(httpContext);
    }
}

Now all that's left is to decorate your controllers/actions with this new attribute:

[MyAuthorize(Roles = "Admin")]
public ActionResult Foo()
{
    ...
}

UPDATE:

As requested in the comments section here's how you could override the HandleUnauthorizedRequest method in the custom authorize attribute so that if the user is not authorized to access a given action he is redirected to some error view instead of the login page:

protected override void HandleUnauthorizedRequest(AuthorizationContext filterContext)
{
    filterContext.Result = new ViewResult
    {
        ViewName = "~/Views/Shared/Unauthorized.cshtml"
    };
}