############################### # # User-agent: * # # list folders robots are not allowed to index # #Disallow: /whatever/folder/ #Disallow: /whatever/else/ #Disallow: /whatever/assets/ # # list specific files robots are not allowed to index # #Disallow: /directory/foo.html Disallow: /editor Disallow: /Bin Disallow: /log_in.aspx # # ###############################