Merge pull request #3393 from Shnoulle/develop_feature3334

Simple robots.txt for server 404 log

mod/robots_txt.php

@@ -0,0 +1,30 @@
+<?php
+/**
+ * @file mod/robots_text.php
+ * @brief Module which returns the default robots.txt
+ * @version 0.1.2
+ */
+
+
+/**
+ * @brief Return default robots.txt when init
+ * @param App $a
+ * @return void
+ */
+function robots_txt_init(App $a)
+{
+
+	/** @var string[] globally disallowed url */
+	$allDisalloweds = array(
+		'/settings/',
+		'/admin/',
+		'/message/',
+	);
+
+	header('Content-Type: text/plain');
+	echo 'User-agent: *'.PHP_EOL;
+	foreach ($allDisalloweds as $disallowed) {
+		echo 'Disallow: '.$disallowed.PHP_EOL;
+	}
+	killme();
+}