# All robots will spider the domain with following directories excluded
User-agent: *
Allow: / 
Disallow: /photos
Disallow: /pics
Disallow: /cgi-bin 
Disallow: /wp-content