What’s the closest thing to Haskell’s typeclasses in OCaml?

It really depends what you want to achieve.

If you are happy with the OCaml polymorphic comparison function (which will not work on cyclic and functional values), you can simply write:

let my_sort l = List.sort Pervasives.compare l

The more generic way to mimic type classes is to use functors:

module type COMPARABLE = sig
  type t
  val compare: t -> t -> int
end

module MySort (C: COMPARABLE) = struct
  let sort l = List.sort C.compare l
end

(* You can now use instantiate the functor *)
module IntAscending = struct
  type t = int
  let compare = (-)
end
module IntDescending = struct
  type t = int
  let compare x y = y - x (* Reverse order *)
end

module SortAsc = MySort(IntAscending)
module SortDesc = MySort(IntDescending)