How can I dim the background when Bottomsheet is displayed, without using Dialog?

You can use this code
1. MainActivity.xml

<android.support.design.widget.CoordinatorLayout
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:id="@+id/main_content"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:fitsSystemWindows="true">

<ScrollView
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    app:layout_behavior="@string/appbar_scrolling_view_behavior">

    <LinearLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:orientation="vertical"
        android:paddingTop="24dp">

        <Button
            android:id="@+id/button_1"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:text="Button 1"
            android:padding="16dp"
            android:layout_margin="8dp"
            android:textColor="@android:color/white"
            android:background="@android:color/holo_green_dark"/>

    </LinearLayout>

</ScrollView>

<View
    android:visibility="gone"
    android:id="@+id/bg"
    android:background="#99000000"
    android:layout_width="match_parent"
    android:layout_height="match_parent"/>

<android.support.v4.widget.NestedScrollView
    android:id="@+id/bottom_sheet"
    android:layout_width="match_parent"
    android:layout_height="350dp"
    android:clipToPadding="true"
    android:background="@android:color/holo_orange_light"
    app:layout_behavior="android.support.design.widget.BottomSheetBehavior"
    >

    <TextView
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:text="aefwea"
        android:padding="16dp"
        android:textSize="16sp"/>

</android.support.v4.widget.NestedScrollView>

 </android.support.design.widget.CoordinatorLayout>

2. MAinActivity.java

   public class MainActivity extends AppCompatActivity implements View.OnClickListener {
   
   private static final String TAG = "MainActivity";
   private BottomSheetBehavior mBottomSheetBehavior;
   View bottomSheet;
   View mViewBg;
   
   @Override
   protected void onCreate(Bundle savedInstanceState) {
       super.onCreate(savedInstanceState);
       setContentView(R.layout.activity_main);
       bottomSheet = findViewById(R.id.bottom_sheet);
       mViewBg = findViewById(R.id.mViewBg);
       Button button1 = (Button) findViewById(R.id.button_1);
       button1.setOnClickListener(this);
       mViewBg.setOnClickListener(this);
       mBottomSheetBehavior = BottomSheetBehavior.from(bottomSheet);
       mBottomSheetBehavior.setBottomSheetCallback(new BottomSheetBehavior.BottomSheetCallback() {
           @Override
           public void onStateChanged(@NonNull View bottomSheet, int newState) {
               if (newState == BottomSheetBehavior.STATE_COLLAPSED)
                   mViewBg.setVisibility(View.GONE);
           }
   
           @Override
           public void onSlide(@NonNull View bottomSheet, float slideOffset) {
               Log.d(TAG, "onSlide: slideOffset" + slideOffset + "");
              mViewBg.setVisibility(View.VISIBLE);
               mViewBg.setAlpha(slideOffset);
           }
       });
   
   }
   
   
   @Override
   public void onClick(View v) {
       switch (v.getId()) {
           case R.id.button_1: {
               mBottomSheetBehavior.setState(BottomSheetBehavior.STATE_EXPANDED);
               break;
           }
           case R.id.bg: {
               mBottomSheetBehavior.setState(BottomSheetBehavior.STATE_COLLAPSED);
               break;
           }
       }
   }
   
   
   @Override
   public boolean dispatchTouchEvent(MotionEvent event) {
       if (event.getAction() == MotionEvent.ACTION_DOWN) {
           if (mBottomSheetBehavior.getState() == BottomSheetBehavior.STATE_EXPANDED) {
               Rect outRect = new Rect();
               bottomSheet.getGlobalVisibleRect(outRect);
               if (!outRect.contains((int) event.getRawX(), (int) event.getRawY())) {
                   mBottomSheetBehavior.setState(BottomSheetBehavior.STATE_COLLAPSED);
                   return true;
               }
   
           }
       }
       return super.dispatchTouchEvent(event);
   }
   
   }