User-agent: * Crawl-delay: 120 Disallow: /check_session* Disallow: /collection_organization/* Disallow: /inventory/* Disallow: /repositories/*/resources/*/tree/* Disallow: /repositories/*/top_containers/* Disallow: /search* Disallow: /staff* Disallow: /oai* Disallow: /advanced_search* Disallow: /assets* User-agent: AhrefsBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Baiduspider Disallow: / User-agent: BLEXBot Disallow: / User-agent: gsa-crawler Disallow: / User-agent: MauiBot Disallow: / User-agent: MegaIndex.ru/2.0 Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: megaIndex.ru Disallow: / User-agent: MJ12bot Disallow: / User-agent: PetalBot Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: SemanticScholarBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SEOkicks Disallow: / User-agent: YandexBot Disallow: / User-agent: WormlyBot Disallow: /