User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /t3lib/
Disallow: /typo3/
Disallow: /typo3_src/
Disallow: /typo3versions/
Disallow: /typo3conf/
Allow: /typo3/sysext/frontend/Resources/Public/*
Allow: /typo3temp/
Allow: /typo3conf/ext/ilanz/Resources/Public/


# Google Crawler
User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /t3lib/
Disallow: /typo3/
Disallow: /typo3_src/
Disallow: /typo3versions/
Disallow: /typo3conf/
Allow: /typo3/sysext/frontend/Resources/Public/*
Allow: /typo3temp/
Allow: /typo3conf/ext/ilanz/Resources/Public/
Crawl-delay: 5

# Yahoo! Crawler
User-agent: Slurp
Disallow: /cgi-bin/
Disallow: /t3lib/
Disallow: /typo3/
Disallow: /typo3_src/
Disallow: /typo3versions/
Disallow: /typo3conf/
Allow: /typo3/sysext/frontend/Resources/Public/*
Allow: /typo3temp/
Allow: /typo3conf/ext/ilanz/Resources/Public/

# allow adsense bot on entire site
#User-agent: Mediapartners-Google*
#Allow: /*.php$
#Allow: /*.js$
#Allow: /*.inc$
#Allow: /*.css$
#Allow: /*.gz$
#Allow: /*.cgi$
#Allow: /*.wmv$
#Allow: /*.cgi$
#Allow: /*.xhtml$
#Allow: /*.php*
#Allow: /*.gif$
#Allow: /*.jpg$
#Allow: /*.png$

# Only allow URLs generated with frontend routing
Disallow: /*?id=*
Disallow: /*&id=*

# L=0 is the default language
Disallow: /*?L=0*
Disallow: /*&L=0*

# Should always be protected, but you know...
Disallow: /*/Private/*
Disallow: /*/Configuration/*

# Disallow all files in /typo3temp/var/
Disallow: /typo3temp/var/*

# Disallow all files in /typo3/
Disallow: /typo3/

# Disallow all kind of sql files
Disallow: *.sql
Disallow: *.sql.gz


# Block known aggressive / unwanted crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: DuckDuckBot
Disallow:
Crawl-delay: 5

User-agent: Baiduspider
Disallow:
Crawl-delay: 10

# Default policy for all others (unknown bots)
User-agent: *
Disallow: /private/
Disallow: /api/
Crawl-delay: 10

Sitemap: https://www.ilanz-glion.ch/sitemap.xml