MatDialog Service Unit Test Angular 6 Error

Testing mat-dialogs can be tricky. I tend to use a spy object for the return from a dialog open (dialogRefSpyObj below) so I can more easily track and control tests. In your case it might look something like the following:

describe('ModalService', () => {
    let modalService: ModalService;
    let dialogSpy: jasmine.Spy;
    let dialogRefSpyObj = jasmine.createSpyObj({ afterClosed : of({}), close: null });
    dialogRefSpyObj.componentInstance = { body: '' }; // attach componentInstance to the spy object...

    beforeEach(() => {
        TestBed.configureTestingModule({
            imports: [MatDialogModule],
            providers: [ModalService]
        });
        modalService = TestBed.get(ModalService);
    });

    beforeEach(() => {
        dialogSpy = spyOn(TestBed.get(MatDialog), 'open').and.returnValue(dialogRefSpyObj);
    });

    it('open modal ', () => {
        modalService.open(TestComponent, '300px');
        expect(dialogSpy).toHaveBeenCalled();

        // You can also do things with this like:
        expect(dialogSpy).toHaveBeenCalledWith(TestComponent, { maxWidth: '100vw' });

        // and ...
        expect(dialogRefSpyObj.afterClosed).toHaveBeenCalled();
    });
});