# /robots.txt for http://anime.mikomi.org/ # Yahoo Feed Seeker's request patterns are really messed up. # What's more, it seems to ignore this, so typically I have it's IPs blocked # by firewall rules. User-agent: YahooFeedSeeker Disallow: * User-agent: YahooFeedSeeker/2.0 Disallow: * User-agent: Slurp Crawl-delay: 10 # Don't index login/logout and editing pages Disallow: /edit/ Disallow: /login.html Disallow: /logout.html Disallow: /create.html Disallow: /editself.html Disallow: /forum/post.html Disallow: /forum/newtopic.html # Nor the mirror of the site that isn't wrapped in templates Disallow: /nowrap/ # Nor webstats Disallow: /stats/ # Nor the pages with series page meta data Disallow: /series_links/ Disallow: /series_indexed/ # Nor the auto generated list of referrers Disallow: /links.html # Nor the search page Disallow: /by-search.html # Nor the opensearch page Disallow: /feeds/opensearch.xml # Nor the wiki history Disallow: /wiki_diff.html Disallow: /wiki_history/ # Nor the infinite sub-categories Disallow: /by-key/*/* # We also stop indexing of popular porn series as, while I'm okay hosting # the reviews, I don't want that to be what the site is known for. # This is by no means a comprehensive list, just the titles that have shown # up as popular in the past. Disallow: /series/323.html Disallow: /series/467.html Disallow: /series/585.html Disallow: /series/632.html Disallow: /series/640.html Disallow: /series/673.html Disallow: /series/744.html Disallow: /series/935.html Disallow: /series/1067.html Disallow: /series/1213.html Disallow: /series/1461.html # And the categories Disallow: /by-key/Hentai Disallow: /by-key/Lolicon Disallow: /by-key/Ecchi Disallow: /by-key/Incest Disallow: /by-key/Hentai.html Disallow: /by-key/Lolicon.html Disallow: /by-key/Ecchi.html Disallow: /by-key/Incest.html User-agent: * # Don't index login/logout and editing pages Disallow: /edit/ Disallow: /login.html Disallow: /logout.html Disallow: /create.html Disallow: /editself.html Disallow: /forum/post.html Disallow: /forum/newtopic.html # Nor the mirror of the site that isn't wrapped in templates Disallow: /nowrap/ # Nor webstats Disallow: /stats/ # Nor the pages with series page meta data Disallow: /series_links/ Disallow: /series_indexed/ # Nor the auto generated list of referrers Disallow: /links.html # Nor the search page Disallow: /by-search.html # Nor the opensearch page Disallow: /feeds/opensearch.xml # Nor the wiki history Disallow: /wiki_diff.html Disallow: /wiki_history/ # Nor the infinite sub-categories Disallow: /by-key/*/* # We also stop indexing of popular porn series as, while I'm okay hosting # the reviews, I don't want that to be what the site is known for. # This is by no means a comprehensive list, just the titles that have shown # up as popular in the past. Disallow: /series/323.html Disallow: /series/467.html Disallow: /series/585.html Disallow: /series/632.html Disallow: /series/640.html Disallow: /series/673.html Disallow: /series/744.html Disallow: /series/935.html Disallow: /series/1067.html Disallow: /series/1213.html Disallow: /series/1461.html # And the categories Disallow: /by-key/Hentai Disallow: /by-key/Lolicon Disallow: /by-key/Ecchi Disallow: /by-key/Incest Disallow: /by-key/Hentai.html Disallow: /by-key/Lolicon.html Disallow: /by-key/Ecchi.html Disallow: /by-key/Incest.html