# Robots exclusion file for http://www.ebi.ac.uk/
# Any questions/comments to es-request@ebi.ac.uk
User-agent: MJ12bot
Disallow: /

User-agent: naverbot
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: discobot 
Disallow: /

#User-agent: msnbot
#Disallow: /

User-agent: EBISearch
Crawl-Delay: 0

User-agent: *
Crawl-Delay: 30
Disallow: /seqdb/confluence
Disallow: /*private*
Disallow: /~
Disallow: /3D
Disallow: /abc
Disallow: /awstats
Disallow: /beta/
Disallow: /biocat/
Disallow: /biocatalogue
Disallow: /bioinformatics
Disallow: /biomodels/papers
Disallow: /cgi-bin/printable
Disallow: /cgi-bin/webservices
Disallow: /cgi-bin/*?
Disallow: /cgi_files
Disallow: /citexplore
Disallow: /come/dags
Disallow: /come/docs
Disallow: /come/entry
Disallow: /come/find
Disallow: /come/path
Disallow: /come/pathrough
Disallow: /Compass
Disallow: /contrib
Disallow: /cpg
Disallow: /Databases/Genome_MOT/
Disallow: /dbases
Disallow: /dbest
Disallow: /~dbinns/documentation/
Disallow: /disease
Disallow: /downloads
Disallow: /dx.doi.org
Disallow: /ebibbs
Disallow: /ebi_docs
Disallow: /ega/dev
Disallow: /embl/Documentation/FAQ/top.html
Disallow: /embl/Documentation/top.html
Disallow: /embnet.news
Disallow: /ena/data/warehouse
Disallow: /ena/data/view
Disallow: /ensembl/biomart
Disallow: /EnsMart
Disallow: /epo
Disallow: /epo-dev
Disallow: /errors
Disallow: /es/doku
Disallow: /es/private_es/
Disallow: /es/private_staff/
Disallow: /euro2004
Disallow: /eurocarb
Disallow: /euprojects
Disallow: /fiv97
Disallow: /flybase
Disallow: /fssp
Disallow: /~gaudan
Disallow: /Genome_MOT
Disallow: /genomes/mot/
Disallow: /Groups/images
Disallow: /Groups/include
Disallow: /hinxton
Disallow: /~holm
Disallow: /images
Disallow: /include
Disallow: /industry/private
Disallow: /industry_SME
Disallow: /info
Disallow: /Information/images
Disallow: /Information/include
Disallow: /Information/local
Disallow: /Information/Seminars/EU1998/
Disallow: /intact/graph2MIF
Disallow: /intact/hierarchView
Disallow: /intact/mine/
Disallow: /intact/search
Disallow: /intact/statisticView
Disallow: /integr8
Disallow: /interpro/internal-tools
Disallow: /interpro/internal-tools/
Disallow: /interpro/ISpy
Disallow: /interpro/RI*
Disallow: /ismb97
Disallow: /local
Disallow: /login
Disallow: /miamexpress/devhelp
Disallow: /~michele/jalview
Disallow: /microarray/ArrayExpress
Disallow: /microarray-as/atlas/qr
Disallow: /microarray/ExpressionProfiler
Disallow: /microarray/General/Internal
Disallow: /microarray/MIAMExpress
Disallow: /microarray/interfaces
Disallow: /_mm
Disallow: /_mmDBScripts
Disallow: /_mmServerScripts
Disallow: /msd.old/
Disallow: /msd-srv/ssm/cgi-bin/
Disallow: /newt
Disallow: /_notes
Disallow: /oib97
Disallow: /oib98
Disallow: /~osklyar
Disallow: /~ouzounis
Disallow: /parasites
Disallow: /powerpoint
Disallow: /profiles
Disallow: /proteincol
Disallow: /protein_profiles
Disallow: /proteomeApplications
Disallow: /PSD
Disallow: /pub
Disallow: /rd
Disallow: /research/cgg
Disallow: /RHdb
Disallow: /scripts
Disallow: /searches/netsearch.html
Disallow: /services.html
Disallow: /services/images
Disallow: /services/include
Disallow: /sift
Disallow: /site_indexer
Disallow: /sites/ebi.ac.uk/files/person/image/Emanuele_Alpi.jpeg
Disallow: /sites/ebi.ac.uk/files/styles/medium/public/person/image/Emanuele_Alpi.jpeg
Disallow: /soaplab
Disallow: /software
Disallow: /SSC
Disallow: /subs
Disallow: /sw
Disallow: /Systems
Disallow: /tc-test
Disallow: /Templates
Disallow: /test
Disallow: /text
Disallow: /textonly
Disallow: /thornton-srv/databases/enzymes
Disallow: /thornton-srv/databases/practicals
Disallow: /thornton-srv/databases/targets
Disallow: /tops
Disallow: /wc2002
Disallow: /webservices/documentation/
Disallow: /xs

#Drupal site
User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
# Taxonomies
Disallow: /taxonomy

#Training site
User-agent: *
Crawl-delay: 10
# Directories
Disallow: /training/includes/
Disallow: /training/misc/
Disallow: /training/modules/
Disallow: /training/profiles/
Disallow: /training/scripts/
Disallow: /training/themes/
# Files
Disallow: /training/CHANGELOG.txt
Disallow: /training/cron.php
Disallow: /training/INSTALL.mysql.txt
Disallow: /training/INSTALL.pgsql.txt
Disallow: /training/install.php
Disallow: /training/INSTALL.txt
Disallow: /training/LICENSE.txt
Disallow: /training/MAINTAINERS.txt
Disallow: /training/update.php
Disallow: /training/UPGRADE.txt
Disallow: /training/xmlrpc.php
# Paths (clean URLs)
Disallow: /training/admin/
Disallow: /training/comment/reply/
Disallow: /training/filter/tips/
Disallow: /training/logout/
Disallow: /training/node/add/
Disallow: /training/search/
Disallow: /training/user/register/
Disallow: /training/user/password/
Disallow: /training/user/login/
# Paths (no clean URLs)
Disallow: /training/?q=admin/
Disallow: /training/?q=comment/reply/
Disallow: /training/?q=filter/tips/
Disallow: /training/?q=logout/
Disallow: /training/?q=node/add/
Disallow: /training/?q=search/
Disallow: /training/?q=user/password/
Disallow: /training/?q=user/register/
Disallow: /training/?q=user/login/
# Nodes
Disallow: /training/Studentships
Disallow: /training/studentships/howtoapply
Disallow: /training/studentships/phdprojects

#Block robots for RDF services, RT#19945
#NB only Google and Yahoo understand wildcards
Disallow: /rdf/services/*/sparql