Added configurable robots.txt file.

Deleted old static file.
Default output depends on app-public setting.
Otherwise can be overidden in `.env` file via `ALLOW_ROBOTS`
Otherwise view file can be customized.

Fixes #779

app/Http/Controllers/HomeController.php

@@ -118,4 +118,20 @@ class HomeController extends Controller
     {
         return view('partials/custom-head-content');
     }
+
+    /**
+     * Show the view for /robots.txt
+     * @return $this
+     */
+    public function getRobots()
+    {
+        $sitePublic = setting('app-public', false);
+        $allowRobots = config('app.allow_robots');
+        if ($allowRobots === null) {
+            $allowRobots = $sitePublic;
+        }
+        return response()
+            ->view('robots', ['allowRobots' => $allowRobots])
+            ->header('Content-Type', 'text/plain');
+    }
 }

config/app.php

@@ -4,12 +4,28 @@ return [
 
     'env' => env('APP_ENV', 'production'),
 
+    /**
+     * Set the default view type for various lists. Can be overridden by user preferences.
+     * This will be used for public viewers and users that have not set a preference.
+     */
     'views' => [
         'books' => env('APP_VIEWS_BOOKS', 'list')
     ],
 
+    /**
+     * Allow <script> tags to entered within page content.
+     * <script> tags are escaped by default.
+     * Even when overridden the WYSIWYG editor may still escape script content.
+     */
     'allow_content_scripts' => env('ALLOW_CONTENT_SCRIPTS', false),
 
+    /**
+     * Override the default behaviour for allowing crawlers to crawl the instance.
+     * May be ignored if view has be overridden or modified.
+     * Defaults to null since, if not set, 'app-public' status used instead.
+     */
+    'allow_robots' => env('ALLOW_ROBOTS', null),
+
     /*
     |--------------------------------------------------------------------------
     | Application Debug Mode

public/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow:

resources/views/robots.blade.php

@@ -0,0 +1,6 @@
+User-agent: *
+@if($allowRobots)
+Disallow:
+@else
+Disallow: /
+@endif

routes/web.php

@@ -2,6 +2,7 @@
 
 Route::get('/translations', 'HomeController@getTranslations');
 Route::get('/icon/{iconName}.svg', 'HomeController@getIcon');
+Route::get('/robots.txt', 'HomeController@getRobots');
 
 // Authenticated routes...
 Route::group(['middleware' => 'auth'], function () {

tests/PublicActionTest.php

@@ -90,4 +90,35 @@ class PublicActionTest extends BrowserKitTest
         $this->dontSee($page->name);
     }
 
+    public function test_robots_effected_by_public_status()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+    }
+
+    public function test_robots_effected_by_setting()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        config()->set('app.allow_robots', true);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+
+        // Check config overrides app-public setting
+        config()->set('app.allow_robots', false);
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow: /");
+    }
+
 }