# entries specific to googlebot crawler - this entry is no longer needed as we allow abstract to all crawlers allowed to crawl our site
User-agent: Googlebot
Disallow: /medline/
Disallow: /search/
Disallow: /user/
Disallow: /addtoreadinglist
Disallow: /citationdownload
Disallow: /claim/
Disallow: /ecommerce/
Disallow: /retrieve/
#Disallow: /webfiles/
Disallow: /*pdf*
#Disallow: /*fulltext*
Disallow: /*abstracts*
Disallow: /article/abstracts
Disallow: /*ppt*
#Allow: /google542652b0c970aa16.html

# entries specific to enforme spider
User-agent: aao-appliance
Disallow: /medline/
Disallow: /search/
Disallow: /user/
Disallow: /addtoreadinglist
Disallow: /citationdownload
Disallow: /claim/
Disallow: /ecommerce/
Disallow: /retrieve/
#Disallow: /webfiles/
Disallow: /article/abstracts

# entries specific to Gatroenterology
User-agent: gsa-crawler (Enterprise; S5-CRES3RTWB4NJA; <a href="mailto:administrator@gastro.org">administrator@gastro.org</a>)
Disallow: /medline/
Disallow: /search/
Disallow: /user/
Disallow: /addtoreadinglist
Disallow: /citationdownload
Disallow: /claim/
Disallow: /ecommerce/
Disallow: /retrieve/
#Disallow: /webfiles/
Disallow: /article/abstracts

# entries specific to AGA
User-agent: aga-crawler-FB3T8L91 (Enterprise; T2-G9KJEYJZFNWAB; agasearch@browsermedia.com)
Disallow: /medline/
Disallow: /search/
Disallow: /user/
Disallow: /addtoreadinglist
Disallow: /citationdownload
Disallow: /claim/
Disallow: /ecommerce/
Disallow: /retrieve/
#Disallow: /webfiles/
Disallow: /article/abstracts

User-agent: Baiduspider
Disallow: / 

# entries for all other crawlers (allow abstracts to all)
User-agent: *
Disallow: /medline/
Disallow: /search/
Disallow: /user/
Disallow: /addtoreadinglist
Disallow: /citationdownload
Disallow: /claim/
Disallow: /ecommerce/
Disallow: /retrieve/
#Disallow: /webfiles/
Disallow: /pdf
Disallow: /article/abstracts
Disallow: /fulltext