# exclusions based on Web Status Search Exclusions # example from http://www.robotstxt.org/wc/norobots.html User-agent: ISYS # address ISYS Spider Disallow: /paguide/chap1.pdf Disallow: /paguide/chap1a4.pdf Disallow: /paguide/chap2.pdf Disallow: /paguide/chap2a4.pdf Disallow: /paguide/chap3.pdf Disallow: /paguide/chap3a4.pdf Disallow: /paguide/chap4.pdf Disallow: /paguide/chap4a4.pdf Disallow: /paguide/chap5.pdf Disallow: /paguide/chap5a4.pdf Disallow: /paguide/chap6.pdf Disallow: /paguide/chap6a4.pdf Disallow: /paguide/chap7.pdf Disallow: /paguide/chap7a4.pdf Disallow: /paguide/chap8.pdf Disallow: /paguide/chap8a4.pdf Disallow: /paguide/chap9.pdf Disallow: /paguide/chap9a4.pdf Disallow: /paguide/chap10.pdf Disallow: /paguide/chap10a4.pdf Disallow: /paguide/chap11.pdf Disallow: /paguide/chap11a4.pdf Disallow: /paguide/chap12.pdf Disallow: /paguide/chap12a4.pdf Disallow: /paguide/chap13.pdf Disallow: /paguide/chap13a4.pdf User-agent: * # address all robots Disallow: /2004/ Disallow: /ads_ebiz/ Disallow: /amplification/ Disallow: /amplificationasst/program/ Disallow: /announce/ Disallow: /apoasst/program/ Disallow: /aqorder/ Disallow: /aventis/ Disallow: /biolum/ Disallow: /branch/ Disallow: /catalog/component.aspx Disallow: /catalog_note/ Disallow: /catalogdown/ Disallow: /cellfree/ Disallow: /cellprolif/program/ Disallow: /cellsig/ Disallow: /cellviability/ Disallow: /citations_submit/ Disallow: /citations/controls/ Disallow: /clonmut/ Disallow: /ctx/program/ Disallow: /dept873/ Disallow: /distrib/ Disallow: /dlrspecialoffer/ Disallow: /dmlib/ Disallow: /downloads/ Disallow: /ena/ Disallow: /exp/ Disallow: /fas/ Disallow: /fas/forms/ Disallow: /figures/ Disallow: /geneticsymp16/registersecure/ Disallow: /genomic/ Disallow: /gifs/ Disallow: /hp2004_content/ Disallow: /hts/ Disallow: /htscelltiterglo/ Disallow: /images/ Disallow: /images_2004/ Disallow: /images10/ Disallow: /images8/ Disallow: /images9/ Disallow: /mastermix/ Disallow: /messages/ Disallow: /midiprep/ Disallow: /moldx/ Disallow: /nap/ Disallow: /nar/ Disallow: /neuro/ Disallow: /neurores_note Disallow: /neuroresource/ Disallow: /paguide/iphone/ Disallow: /paguide/iphonebeta/ Disallow: /pcr/us/ Disallow: /pcrref/ Disallow: /printers/ Disallow: /profiles/survey/ Disallow: /prolink/ Disallow: /propopup/ Disallow: /protcards/ Disallow: /protectme/ Disallow: /proteolink/ Disallow: /pure/ Disallow: /puredna/ Disallow: /pureplantdna Disallow: /qualitymonitoring/ Disallow: /readit/probedesign/program/ Disallow: /readitprobe/ Disallow: /register/ Disallow: /reguide/ Disallow: /rnafam/ Disallow: /rtpcr/ Disallow: /scripts/ Disallow: /search/ Disallow: /serverdown/ Disallow: /sirnadesigner/program/ Disallow: /siteserver/ Disallow: /stoprnase/ Disallow: /stylesheets/ Disallow: /surveys/ Disallow: /taq/ Disallow: /taqlegal/ Disallow: /tbscustom/ Disallow: /welcome/ Disallow: /widgets/