git filter-branch --force --index-filter 'git rm --cached -r --ignore-unmatch directoryToRemove' --prune-empty -- --all
git push origin master --force

Note: these days git filter-repo is the recommended tool for this job.