You could use df._get_numeric_data() to get numeric columns and then find out categorical columns
In [66]: cols = df.columns
In [67]: num_cols = df._get_numeric_data().columns
In [68]: num_cols
Out[68]: Index([u'0', u'1', u'2'], dtype="object")
In [69]: list(set(cols) - set(num_cols))
Out[69]: ['3', '4']