#
# robots.txt
#

# Disallow commercial bots to prevent ad fraud, see DISC-2117
User-agent: psbot
User-agent: Yandex
User-agent: PetalBot 
User-agent: Mail.RU_Bot
User-agent: MegaIndex
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
User-agent: Bytespider
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: proximic
User-agent: ADmantX
User-agent: Seekport Crawler
User-agent: SEMrushBot
User-agent: BLEXBot
User-agent: MJ12bot
User-agent: dotbot
User-agent: BUbiNG
User-agent: omgili
User-agent: Exabot
User-agent: BLP_bbot/0.1
User-agent: BLP_bbot
User-agent: sentibot
User-agent: Meltawer
User-agent: Digimind
User-agent: Knowings
User-agent: Sindup
User-agent: Cision
User-agent: Talkwater
User-agent: TurnitinBot
User-agent: ConveraCrawler
User-agent: Jetbot
User-agent: NewsNow
User-agent: kbcrawl
User-agent: AmiSoftware
User-agent: Newzbin
User-agent: Ask n read
User-agent: Qwam content intelligence
User-agent: Zite
User-agent: flipboard
User-agent: Youmag
User-agent: Synthesio
User-agent: trendybuzz
User-agent: spotter
User-agent: scoop.it
User-agent: trendybuzz
User-agent: linkfluence
User-agent: 5emeRue
User-agent: spotter
User-agent: Augure
User-agent: Corporama
User-agent: grub-client
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
User-agent: k2spider
User-agent: libwww
User-agent: wget
User-agent: 5erue
User-agent: adequat
User-agent: adequat-systems
User-agent: auramundi
User-agent: coexel
User-agent: ellisphere
User-agent: leadbox
User-agent: mention
User-agent: Moreover
User-agent: mytwip
User-agent: NewsNow
User-agent: Newzbin
User-agent: opinion-tracker
User-agent: proxem
User-agent: score3
User-agent: trendeo
User-agent: vecteurplus
User-agent: verticalsearch
User-agent: vsw
User-agent: winello
User-agent: Fetch
User-agent: infoseek
User-agent: MSIECrawler
User-agent: Offline Explorer
User-agent: sitecheck.internetseer.com
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebCopier
User-agent: WebStripper
User-agent: Zealbot
User-agent: asknread.com
User-agent: ellisphere
User-agent: spotter
User-Agent: omgilibot
Disallow: /

# Allow crawling for other bots
User-agent: *
Allow: /
Sitemap: https://www.femina.ch/sitemap.xml

# Files
User-agent: *
Disallow: /cpresources/
Disallow: /vendor/
Disallow: /.env
Disallow: /version.txt

# Paths (clean URLs)
Disallow: /admin/
Disallow: /cmsadm/
Disallow: /recherche/
Disallow: /api/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /index.php?p=admin/install

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/