User-agent: * # match any robot name Disallow: /admin/ # exclude admin folder Disallow: /assets/ # don't allow robots into the assets folder Disallow: /engine/ # exclude engine folder Disallow: /setup/ # exclude setup folder