# /robots.txt file for http://www.esrl.noaa.gov User-agent: * Disallow: /img # PSD Disallow: /psd/psd3/satres/data/images/ Disallow: /psd/psd3/arctic/sheba/netcdf/ Disallow: /psd/psd3/arctic/nsa/netcdf/ # CSD Disallow: /csd/cgi-bin Disallow: /csd/acgi-bin Disallow: /csd/css Disallow: /csd/dbs Disallow: /csd/internal Disallow: /csd/nav Disallow: /csd/aqrs/presentations Disallow: /csd/metproducts Disallow: /csd/flexpart Disallow: /csd/modeleval Disallow: /csd/ICARTT/analysis SITEMAP: http://www.esrl.noaa.gov/esrl_sitemap.xml.gz #examples # Allows all search # User-agent: something # Disallow: # Prohibit all search # User-agent: something # Disallow: /