# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 #
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-Agent: *
 # Disallow: /

 # Add a 1 second delay between successive requests to the same server, limits resources used by crawler
 # Only some crawlers respect this setting, e.g. Googlebot does not
 # Crawl-delay: 1

# ban Yandex
User-agent: Yandex
Disallow: /
Crawl-delay: 20   # delay 20sec

# ban YandexBot
User-agent: YandexBot
Disallow: /
Crawl-delay: 20  # delay 20sec

 # Based on details in https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/routes.rb, https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/routing, and using application
 User-Agent: *
 Disallow: /gitlab/autocomplete/users
 Disallow: /gitlab/search
 Disallow: /gitlab/api
 Disallow: /gitlab/admin
 Disallow: /gitlab/profile
 Disallow: /gitlab/dashboard
 Disallow: /gitlab/projects/new
 Disallow: /gitlab/groups/new
 Disallow: /gitlab/groups/*/edit
 Disallow: /gitlab/users
 Disallow: /gitlab/help
 # Only specifically allow the Sign In page to avoid very ugly search results
 Allow: /gitlab/users/sign_in

 # Global snippets
 User-Agent: *
 Disallow: /gitlab/s/
 Disallow: /gitlab/snippets/new
 Disallow: /gitlab/snippets/*/edit
 Disallow: /gitlab/snippets/*/raw

 # Project details
 User-Agent: *
 Disallow: /gitlab/*/*.git
 Disallow: /gitlab/*/*/fork/new
 Disallow: /gitlab/*/*/repository/archive*
 Disallow: /gitlab/*/*/activity
 Disallow: /gitlab/*/*/new
 Disallow: /gitlab/*/*/edit
 Disallow: /gitlab/*/*/raw
 Disallow: /gitlab/*/*/blame
 Disallow: /gitlab/*/*/commits/*/*
 Disallow: /gitlab/*/*/commit/*.patch
 Disallow: /gitlab/*/*/commit/*.diff
 Disallow: /gitlab/*/*/compare
 Disallow: /gitlab/*/*/branches/new
 Disallow: /gitlab/*/*/tags/new
 Disallow: /gitlab/*/*/network
 Disallow: /gitlab/*/*/graphs
 Disallow: /gitlab/*/*/milestones/new
 Disallow: /gitlab/*/*/milestones/*/edit
 Disallow: /gitlab/*/*/issues/new
 Disallow: /gitlab/*/*/issues/*/edit
 Disallow: /gitlab/*/*/merge_requests/new
 Disallow: /gitlab/*/*/merge_requests/*.patch
 Disallow: /gitlab/*/*/merge_requests/*.diff
 Disallow: /gitlab/*/*/merge_requests/*/edit
 Disallow: /gitlab/*/*/merge_requests/*/diffs
 Disallow: /gitlab/*/*/project_members/import
 Disallow: /gitlab/*/*/labels/new
 Disallow: /gitlab/*/*/labels/*/edit
 Disallow: /gitlab/*/*/wikis/*/edit
 Disallow: /gitlab/*/*/snippets/new
 Disallow: /gitlab/*/*/snippets/*/edit
 Disallow: /gitlab/*/*/snippets/*/raw
 Disallow: /gitlab/*/*/deploy_keys
 Disallow: /gitlab/*/*/hooks
 Disallow: /gitlab/*/*/services
 Disallow: /gitlab/*/*/protected_branches
 Disallow: /gitlab/*/*/uploads/
 Disallow:






Sitemap: https://fishreg.jrc.ec.europa.eu/sitemap.xml