#
#  Site had a problem with entry process faults -- too many PHP threads
#  Decided that the problem was crawlers invoking lots of commenter-thread and commenter-archive searches
#  Those are heavy on data base usage and take a "long time" to complete
#  Warn crawlers away from everything related to State of the Discussion
#  Crawl-delay turned out to be unnecessary
#  The wp-admin entries are what WordPress puts in their default virtual robots.txt
#    MEC, 2/2/2021
#

User-agent: *
Crawl-delay: 500
Disallow: /state-of-the-discussion*
Disallow: /state-of-the-discussion*
Disallow: /sotd-test*
Disallow: /sotd-test*
Disallow: /commenter-thread*
Disallow: /commenter-archive*
Disallow: /*/commenter-thread*
Disallow: /*/commenter-archive*
Disallow: /tag/*
Disallow: /blog/*
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php