If you don’t want to make a custom panel as suggested in @Sinatr’s answer you can draw the border yourself:
private void panel1_Paint(object sender, PaintEventArgs e)
{
ControlPaint.DrawBorder(e.Graphics, this.panel1.ClientRectangle, Color.DarkBlue, ButtonBorderStyle.Solid);
}