ASP.NET MVC Authorize Attribute to launch a modal?

In this case you can use a custom action filter attribute that opens a popup if the user is not authorized.
In this action filter just check if user is logged in and add a boolean value to the ViewData collection.
Aplly the attribute on the controller's action.
Then in the master page add conditional rendering of code that opens the popup.

The code for the attribute:

[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = true, AllowMultiple = true)]
public class PopupAuthorizeAttribute : AuthorizeAttribute
{
    private void CacheValidateHandler(HttpContext context, object data, ref HttpValidationStatus validationStatus)
    {
        validationStatus = this.OnCacheAuthorization(new HttpContextWrapper(context));
    }

    public override void OnAuthorization(AuthorizationContext filterContext)
    {
        bool isAuthorized = false;
        if (filterContext == null)
        {
            throw new ArgumentNullException("filterContext");
        }
        if (this.AuthorizeCore(filterContext.HttpContext))
        {
            HttpCachePolicyBase cache = filterContext.HttpContext.Response.Cache;
            cache.SetProxyMaxAge(new TimeSpan(0L));
            cache.AddValidationCallback(new HttpCacheValidateHandler(this.CacheValidateHandler), null);
            isAuthorized = true;
        }

        filterContext.Controller.ViewData["OpenAuthorizationPopup"] = !isAuthorized;
    }
}

In the master page or other common view add conditional rendering:

<% if((bool)(ViewData["OpenAuthorizationPopup"] ?? true)) { %>
 ...Your code to open the popup here...
<% } %>