RedirectToAction is just a helper method to construct a RedirectToRouteResult(), so what you do is simply create a new RedirectToRouteResult() passing along a RouteValueDictionary() with values for your action.
Complete sample based on code from @Domenic in the comment below:
public class IsGuestAttribute: ActionFilterAttribute
{
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
if (!Ctx.User.IsGuest)
{
filterContext.Result = new RedirectToRouteResult(
new RouteValueDictionary
{
{ "controller", "Home" },
{ "action", "Index" }
});
}
}
}