#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html
#
User-agent: *
Crawl-delay: 240
#
Sitemap: http://mobikefed.org/sitemap.xml
#
#Allows - only recognized by Googlebot and maybe a few others, they must come first per http://en.wikipedia.org/wiki/Robots_exclusion_standard#Allow_directive
Allow: /tags/news
Allow: /tags/safe-routes-school
Allow: /tags/cycling-tips-and-stories
Allow: /tags/TourOfMissouri
Allow: /tags/advocacy-alerts
Allow: /archive
Allow: /events
# Directories
#Disallow: /blackwhole/
#Disallow: /blackwhole.html
Disallow: /tags/news?page=
Disallow: /tags/safe-routes-school?page=
Disallow: /tags/cycling-tips-and-stories?page=
Disallow: /tags/TourOfMissouri?page=
Disallow: /tags/advocacy-alerts?page=
Disallow: /events/
Disallow: /taxonomy/
Disallow: /facility/
Disallow: /company/
Disallow: /url/
Disallow: /modules/
Disallow: /user/
Disallow: /usr/
Disallow: /tags/
Disallow: /category/tags/
Disallow: /browsecontent/
Disallow: /social-tags/
Disallow: /sports-event/
Disallow: /province-or-state/
Disallow: /holiday/
Disallow: /industry-term/
Disallow: /published-medium/
Disallow: /organization/
Disallow: /position/
Disallow: /region/
Disallow: /person/
Disallow: /city/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /comment/
Disallow: /cart
#
# Files
#
# These list sponsors & supporters by name, so we'd like to keep them out of web indexes & searches
Disallow: /2013/04/bicycle-and-pedestrian-day-capitol-and-legislative-session-2013-sponsors-and-supporters
Disallow: /mo-donors-supporters
Disallow: /2012-Key-Donors
Disallow: /2013-Key-Donors
Disallow: /2014-Key-Donors
Disallow: /2015-Key-Donors
Disallow: /2016-Key-Donors
#
Disallow: /boost_stats.php
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /logout/
Disallow: /node/add/
Disallow: /node/add/calendar
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /hidden
Disallow: /node/
Disallow: /*/devel
Disallow: /*/edit
Disallow: /*/bikeleague.org
Disallow: /*/add
Disallow: /*/token
Disallow: /*/render
Disallow: /congress
Disallow: /search404
#
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/


User-agent: bingbot
Crawl-delay: 300
#Bingbot is totally aggressive and kills bandwith/cpu/SQL, so we restrict it 
#tightly to just a few pages
#
Sitemap: http://mobikefed.org/sitemap.xml
#
#Allows - only recognized by Googlebot and maybe a few others, they must come first per http://en.wikipedia.org/wiki/Robots_exclusion_standard#Allow_directive
Allow: /tags/news
Allow: /tags/safe-routes-school
Allow: /tags/cycling-tips-and-stories
Allow: /tags/TourOfMissouri
Allow: /tags/advocacy-alerts
Allow: /archive
Allow: /archive/2014
# Directories
#Disallow: /blackwhole/
#Disallow: /blackwhole.html
Disallow: /archive/
Disallow: /topics
Disallow: /tags/news?page=
Disallow: /tags/safe-routes-school?page=
Disallow: /tags/cycling-tips-and-stories?page=
Disallow: /tags/TourOfMissouri?page=
Disallow: /tags/advocacy-alerts?page=
Disallow: /archive
Disallow: /topics
Disallow: /2002/
Disallow: /2003/
Disallow: /2004/
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/
Disallow: /2012/
Disallow: /2013/
Disallow: /taxonomy/
Disallow: /facility/
Disallow: /company/
Disallow: /url/
Disallow: /modules/
Disallow: /user/
Disallow: /usr/
Disallow: /tags/
Disallow: /category/tags/
Disallow: /browsecontent/
Disallow: /social-tags/
Disallow: /sports-event/
Disallow: /province-or-state/
Disallow: /holiday/
Disallow: /industry-term/
Disallow: /published-medium/
Disallow: /position/
Disallow: /organization/
Disallow: /region/
Disallow: /person/
Disallow: /city/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /comment/
Disallow: /events/
Disallow: /cart
#
# Files
#
# These list sponsors & supporters by name, so we'd like to keep them out of web indexes & searches
Disallow: /2013/04/bicycle-and-pedestrian-day-capitol-and-legislative-session-2013-sponsors-and-supporters
Disallow: /mo-donors-supporters
Disallow: /2012-Key-Donors
Disallow: /2013-Key-Donors
Disallow: /2014-Key-Donors
Disallow: /2015-Key-Donors
Disallow: /2016-Key-Donors
#
Disallow: /boost_stats.php
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /logout/
Disallow: /node/add/
Disallow: /node/add/calendar
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /hidden
Disallow: /node/
Disallow: /*/devel
Disallow: /*/edit
Disallow: /*/bikeleague.org
Disallow: /*/add
Disallow: /*/token
Disallow: /*/render
Disallow: /congress
Disallow: /search404
#
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/