# http://www.searchengineworld.com/cgi-bin/robotcheck.cgi User-agent: panscient.com User-agent: httpunit User-agent: RSSOwl User-agent: AUDIOVOX User-agent: ImageVisu User-agent: mindUpBot User-agent: Orbbot User-agent: curl User-agent: AwarioSmartBot User-agent: Twitterbot User-agent: python User-agent: GIEX User-agent: oBot User-agent: Scrapy User-agent: cortex User-agent: InfoTigerBot Disallow: / User-agent: MauiBot User-agent: letsencrypt User-agent: adscanner Crawl-delay: 999 User-agent: IonCrawl User-agent: PetalBot User-agent: panscient User-agent: Mediatoolkitbot User-agent: mediatoolkit User-agent: BingPreview User-agent: YandexBot User-agent: Uptimebot User-agent: adscanner User-agent: MauiBot User-agent: letsencrypt User-agent: DotBot User-agent: opensiteexplorer User-agent: moz # qwant.com User-agent: Qwantify # Impressum+Adress-Scanner: Michael Brunnbauer, München # netEstate NE Crawler (+http://www.website-datenbank.de User-agent: netEstate User-agent: Riddler User-agent: Cliqzbot User-agent: Barkrowler User-agent: netEstate User-agent: baiduspider User-agent: ICCrawler - iCjobs User-agent: Linguee User-agent: seokicks User-agent: WBSearchBot User-agent: Yandex User-agent: memoryBot User-agent: Wotbox User-agent: BUbiNG User-agent: SemrushBot User-agent: 008 User-agent: MetaJobBot User-agent: SeznamBot User-agent: seoscanners User-agent: Qwantify User-agent: spbot User-agent: Java User-agent: MJ12bot User-agent: AhrefsBot User-agent: commoncrawl User-agent: CCBot Disallow: / User-agent: 008 User-agent: MetaJobBot User-agent: seoscanners User-agent: Qwantify User-agent: spbot User-agent: Java User-agent: MJ12bot User-agent: AhrefsBot Disallow: *.pdf$ User-agent: * Disallow: /kontakt1.htm Disallow: /internat/kontakt1.htm Disallow: kontakt1.htm Disallow: *.gif$ Disallow: *.jpg$ User-agent: * Disallow: /~ Disallow: /ad87/ Disallow: /certs/ Disallow: /cgi-bin/ Disallow: /galerie1/ Disallow: /icons/ Disallow: /_intern/ Disallow: /images/ Disallow: /download/ Disallow: /galerie1/ Disallow: /galerie2/ Disallow: /bm-data/ Disallow: /logs/ Disallow: /perl/ Disallow: /_pgp-key/ Disallow: /dummie-robot-test/ Disallow: /_dummie-robot-test/ Disallow: /ssi/ Disallow: /stats/ Disallow: /statistics/ Disallow: /plesk-stat/ Disallow: /test/ Disallow: /usage/ Disallow: /users/ Disallow: /webalizer/ Disallow: /web/~ Disallow: /web/ad87/ Disallow: /web/certs/ Disallow: /web/galerie1/ Disallow: /web/icons/ Disallow: /web/images/ Disallow: /web/perl/ Disallow: /web/ssi/ Disallow: /web/stats/ Disallow: /web/test/ Disallow: _intern Disallow: images Disallow: download Disallow: galerie1 Disallow: galerie2 Disallow: bm-data Disallow: logs Disallow: perl Disallow: stats Disallow: statistics Disallow: plesk-stat Disallow: _pgp-key Disallow: test Disallow: usage Disallow: users Disallow: cgi-bin # "xxx yyy" ok? # ? User-agent: libwww-perl/5.52 # fpcr001.sac2.fastsearch.net User-agent: Asian User-agent: Atomic User-agent: botzilla User-agent: bulk User-agent: bulkmarketer User-agent: bumblebee User-agent: cgi-mailer-bounces User-agent: Collector User-agent: EmailCollector User-agent: email User-agent: e-mail User-agent: FalconEyes # User-agent: Googlebot # User-agent: FAST-WebCrawler # User-agent: fastsearch.net User-agent: formmail.pl User-agent: formmail.cgi User-agent: GagaRobot User-agent: Hotbar # User-agent: ia_archiver User-agent: iCcrawler User-agent: Microsoft URL Control User-agent: MSIECrawler User-agent: Outpost User-agent: post User-agent: rabaz # User-agent: Scooter-W3.1.2 User-agent: sitecheck # User-agent: internetseer # User-agent: SlySearch User-agent: warezcrawler User-agent: Whizbang User-agent: WMP/1.0 User-agent: ZyBorg User-agent: VoilaBot # User-agent: msnbot # User-agent: Nutch User-agent: MMCrawler User-agent: MJ12 Disallow: / # inktomisearch.com dreht durch, yahoo-slurp: # User-agent: Yahoo # User-agent: Slurp # User-agent: googlebot # Crawl-delay: 33 User-agent: * Crawl-delay: 14