Start datasette.json, re-add --config, rm settings.json

The first step in defining the new `datasette.json/yaml` configuration mechanism.

Refs #2093, #2143, #493

datasette/app.py

@@ -242,6 +242,7 @@ def __init__(
         cache_headers=True,
         cors=False,
         inspect_data=None,
+        config=None,
         metadata=None,
         sqlite_extensions=None,
         template_dir=None,
@@ -316,16 +317,27 @@ def __init__(
             )
         self.cache_headers = cache_headers
         self.cors = cors
+        config_files = []
         metadata_files = []
         if config_dir:
             metadata_files = [
                 config_dir / filename
                 for filename in ("metadata.json", "metadata.yaml", "metadata.yml")
                 if (config_dir / filename).exists()
             ]
+            config_files = [
+                config_dir / filename
+                for filename in ("datasette.json", "datasette.yaml", "datasette.yml")
+                if (config_dir / filename).exists()
+            ]
         if config_dir and metadata_files and not metadata:
             with metadata_files[0].open() as fp:
                 metadata = parse_metadata(fp.read())
+
+        if config_dir and config_files and not config:
+            with config_files[0].open() as fp:
+                config = parse_metadata(fp.read())
+
         self._metadata_local = metadata or {}
         self.sqlite_extensions = []
         for extension in sqlite_extensions or []:
@@ -344,17 +356,19 @@ def __init__(
         if config_dir and (config_dir / "static").is_dir() and not static_mounts:
             static_mounts = [("static", str((config_dir / "static").resolve()))]
         self.static_mounts = static_mounts or []
-        if config_dir and (config_dir / "config.json").exists():
-            raise StartupError("config.json should be renamed to settings.json")
-        if config_dir and (config_dir / "settings.json").exists() and not settings:
-            settings = json.loads((config_dir / "settings.json").read_text())
-            # Validate those settings
-            for key in settings:
-                if key not in DEFAULT_SETTINGS:
-                    raise StartupError(
-                        "Invalid setting '{}' in settings.json".format(key)
-                    )
-        self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
+        if config_dir and (config_dir / "datasette.json").exists() and not config:
+            config = json.loads((config_dir / "datasette.json").read_text())
+
+        config = config or {}
+        config_settings = config.get("settings") or {}
+
+        # validate "settings" keys in datasette.json
+        for key in config_settings:
+            if key not in DEFAULT_SETTINGS:
+                raise StartupError("Invalid setting '{}' in datasette.json".format(key))
+
+        # CLI settings should overwrite datasette.json settings
+        self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
         if self.setting("num_sql_threads") == 0:

datasette/cli.py

@@ -50,46 +50,6 @@
     pass
 
 
-class Config(click.ParamType):
-    # This will be removed in Datasette 1.0 in favour of class Setting
-    name = "config"
-
-    def convert(self, config, param, ctx):
-        if ":" not in config:
-            self.fail(f'"{config}" should be name:value', param, ctx)
-            return
-        name, value = config.split(":", 1)
-        if name not in DEFAULT_SETTINGS:
-            msg = (
-                OBSOLETE_SETTINGS.get(name)
-                or f"{name} is not a valid option (--help-settings to see all)"
-            )
-            self.fail(
-                msg,
-                param,
-                ctx,
-            )
-            return
-        # Type checking
-        default = DEFAULT_SETTINGS[name]
-        if isinstance(default, bool):
-            try:
-                return name, value_as_boolean(value)
-            except ValueAsBooleanError:
-                self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx)
-                return
-        elif isinstance(default, int):
-            if not value.isdigit():
-                self.fail(f'"{name}" should be an integer', param, ctx)
-                return
-            return name, int(value)
-        elif isinstance(default, str):
-            return name, value
-        else:
-            # Should never happen:
-            self.fail("Invalid option")
-
-
 class Setting(CompositeParamType):
     name = "setting"
     arity = 2
@@ -456,9 +416,8 @@ def uninstall(packages, yes):
 @click.option("--memory", is_flag=True, help="Make /_memory database available")
 @click.option(
     "--config",
-    type=Config(),
-    help="Deprecated: set config option using configname:value. Use --setting instead.",
-    multiple=True,
+    type=click.File(mode="r"),
+    help="Path to JSON/YAML Datasette configuration file",
 )
 @click.option(
     "--setting",
@@ -568,6 +527,8 @@ def serve(
         reloader = hupper.start_reloader("datasette.cli.serve")
         if immutable:
             reloader.watch_files(immutable)
+        if config:
+            reloader.watch_files([config.name])
         if metadata:
             reloader.watch_files([metadata.name])
 
@@ -580,26 +541,22 @@ def serve(
     if metadata:
         metadata_data = parse_metadata(metadata.read())
 
-    combined_settings = {}
+    config_data = None
     if config:
-        click.echo(
-            "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead",
-            err=True,
-        )
-        combined_settings.update(config)
-    combined_settings.update(settings)
+        config_data = parse_metadata(config.read())
 
     kwargs = dict(
         immutables=immutable,
         cache_headers=not reload,
         cors=cors,
         inspect_data=inspect_data,
+        config=config_data,
         metadata=metadata_data,
         sqlite_extensions=sqlite_extensions,
         template_dir=template_dir,
         plugins_dir=plugins_dir,
         static_mounts=static,
-        settings=combined_settings,
+        settings=dict(settings),
         memory=memory,
         secret=secret,
         version_note=version_note,

docs/cli-reference.rst

@@ -112,8 +112,7 @@ Once started you can access it at ``http://localhost:8001``
       --static MOUNT:DIRECTORY        Serve static files from this directory at
                                       /MOUNT/...
       --memory                        Make /_memory database available
-      --config CONFIG                 Deprecated: set config option using
-                                      configname:value. Use --setting instead.
+      --config FILENAME               Path to JSON/YAML Datasette configuration file
       --setting SETTING...            Setting, see
                                       docs.datasette.io/en/stable/settings.html
       --secret TEXT                   Secret used for signing secure values, such as

docs/configuration.rst

@@ -0,0 +1,10 @@
+.. _configuration:
+
+Configuration
+========
+
+Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more.
+
+To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag:
+
+    datasette mydatabase.db --config datasette.yaml

docs/settings.rst

@@ -47,9 +47,9 @@ Datasette will detect the files in that directory and automatically configure it
 The files that can be included in this directory are as follows. All are optional.
 
 * ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette
+* ``datasette.json`` - :ref:`configuration` for the Datasette instance
 * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well
 * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running
-* ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs
 * ``templates/`` - a directory containing :ref:`customization_custom_templates`
 * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off`
 * ``static/`` - a directory containing static files - these will be served from ``/static/filename.txt``, see :ref:`customization_static_files`

tests/test_cli.py

@@ -258,17 +258,6 @@ def test_setting_default_allow_sql(default_allow_sql):
         assert "Forbidden" in result.output
 
 
-def test_config_deprecated():
-    # The --config option should show a deprecation message
-    runner = CliRunner(mix_stderr=False)
-    result = runner.invoke(
-        cli, ["--config", "allow_download:off", "--get", "/-/settings.json"]
-    )
-    assert result.exit_code == 0
-    assert not json.loads(result.output)["allow_download"]
-    assert "will be deprecated in" in result.stderr
-
-
 def test_sql_errors_logged_to_stderr():
     runner = CliRunner(mix_stderr=False)
     result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"])

tests/test_config_dir.py

@@ -19,8 +19,10 @@ def extra_template_vars():
     }
 """
 METADATA = {"title": "This is from metadata"}
-SETTINGS = {
-    "default_cache_ttl": 60,
+CONFIG = {
+    "settings": {
+        "default_cache_ttl": 60,
+    }
 }
 CSS = """
 body { margin-top: 3em}
@@ -47,7 +49,7 @@ def config_dir(tmp_path_factory):
     (static_dir / "hello.css").write_text(CSS, "utf-8")
 
     (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8")
-    (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8")
+    (config_dir / "datasette.json").write_text(json.dumps(CONFIG), "utf-8")
 
     for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"):
         db = sqlite3.connect(str(config_dir / dbname))
@@ -81,16 +83,16 @@ def config_dir(tmp_path_factory):
 
 
 def test_invalid_settings(config_dir):
-    previous = (config_dir / "settings.json").read_text("utf-8")
-    (config_dir / "settings.json").write_text(
-        json.dumps({"invalid": "invalid-setting"}), "utf-8"
+    previous = (config_dir / "datasette.json").read_text("utf-8")
+    (config_dir / "datasette.json").write_text(
+        json.dumps({"settings": {"invalid": "invalid-setting"}}), "utf-8"
     )
     try:
         with pytest.raises(StartupError) as ex:
             ds = Datasette([], config_dir=config_dir)
-        assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json"
+        assert ex.value.args[0] == "Invalid setting 'invalid' in datasette.json"
     finally:
-        (config_dir / "settings.json").write_text(previous, "utf-8")
+        (config_dir / "datasette.json").write_text(previous, "utf-8")
 
 
 @pytest.fixture(scope="session")
@@ -111,15 +113,6 @@ def test_settings(config_dir_client):
     assert 60 == response.json["default_cache_ttl"]
 
 
-def test_error_on_config_json(tmp_path_factory):
-    config_dir = tmp_path_factory.mktemp("config-dir")
-    (config_dir / "config.json").write_text(json.dumps(SETTINGS), "utf-8")
-    runner = CliRunner(mix_stderr=False)
-    result = runner.invoke(cli, [str(config_dir), "--get", "/-/settings.json"])
-    assert result.exit_code == 1
-    assert "config.json should be renamed to settings.json" in result.stderr
-
-
 def test_plugins(config_dir_client):
     response = config_dir_client.get("/-/plugins.json")
     assert 200 == response.status