launchpad-reviewers team mailing list archive

[Simone Pelosi <mp+446054@xxxxxxxxxxxxxxxxxx>]

Simone Pelosi has proposed merging ~pelpsi/txpkgupload:txpkgupload-charm into txpkgupload:master.

Commit message:
Txpkgupload charm created
    
Listens to FTP and SFTP requests, and processes uploads files into a
temporary directory
This is a subordinate charm, so it will need a master that provides a
`upload-queue-processor` interface to be deployed.

launchpad-layer source-commit should be updated when the related MP is merged.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~pelpsi/txpkgupload/+git/txpkgupload/+merge/446054
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~pelpsi/txpkgupload:txpkgupload-charm into txpkgupload:master.

diff --git a/charm/txpkgupload/README.md b/charm/txpkgupload/README.md
new file mode 100644
index 0000000..74b90e5
--- /dev/null
+++ b/charm/txpkgupload/README.md
@@ -0,0 +1,23 @@
+# TX (Twisted) Package Upload
+
+Listens to FTP and SFTP requests, and processes uploads files into a
+temporary directory
+
+This is a subordinate charm, so it will need a master that provides a 
+`upload-queue-processor` interface to be deployed. The master should have in
+their `metadata.yaml` something on the lines of:
+
+```yaml
+provides:
+  upload-queue-processor:
+    interface: upload-queue-processor
+    scope: container
+```
+
+As this is a subordinate charm, it will be deployed to the same machine as its
+master. The charm will not deploy before the relation to its master is set.
+You will need the following relations:
+
+```shell
+juju relate txpkgupload <name of master application>
+```
diff --git a/charm/txpkgupload/charmcraft.yaml b/charm/txpkgupload/charmcraft.yaml
new file mode 100644
index 0000000..e49a71f
--- /dev/null
+++ b/charm/txpkgupload/charmcraft.yaml
@@ -0,0 +1,60 @@
+type: charm
+bases:
+  - build-on:
+    - name: ubuntu
+      channel: "20.04"
+      architectures: [amd64]
+    run-on:
+    - name: ubuntu
+      channel: "20.04"
+      architectures: [amd64]
+parts:
+  charm-wheels:
+    source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+    source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": charm-wheels/
+    prime:
+      - "-charm-wheels"
+  ols-layers:
+    source: https://git.launchpad.net/ols-charm-deps
+    source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": layers/
+    stage:
+      - layers
+    prime:
+      - "-layers"
+  launchpad-layers:
+    after:
+      - ols-layers
+    source: https://git.launchpad.net/~ines-almeida/launchpad-layers
+    source-commit: "685b45e63ebb56cf505186be1ebba7d914ce60e9"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      upload-queue-processor: layers/interface/upload-queue-processor
+    stage:
+      - layers
+    prime:
+      - "-layers"
+  txpkgupload:
+    after:
+      - charm-wheels
+      - launchpad-layers
+    source: .
+    plugin: reactive
+    build-snaps: [charm/2.x/stable]
+    build-packages: [libpq-dev]
+    build-environment:
+      - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+      - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface
+      - PIP_NO_INDEX: "true"
+      - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/txpkgupload/config.yaml b/charm/txpkgupload/config.yaml
new file mode 100644
index 0000000..aba582e
--- /dev/null
+++ b/charm/txpkgupload/config.yaml
@@ -0,0 +1,25 @@
+options:
+  ftp_port:
+    type: int
+    default: 2221
+    description: The port to run the FTP server on.
+  sftp_authentication_endpoint:
+    type: string
+    default: http://xmlrpc-private.launchpad.dev:8087/authserver
+    description: The URL of the XML-RPC endpoint for authentication handling of SSH users.
+  sftp_host_key_private:
+    type: string
+    default: src/txpkgupload/tests/txpkgupload-sftp
+    description: The absolute path to the private key used for the SFTP server.
+  sftp_host_key_public:
+    type: string
+    default: src/txpkgupload/tests/txpkgupload-sftp.pub
+    description: The absolute path to the public key used for the SFTP server.
+  sftp_port:
+    type: string
+    default: tcp:5023
+    description: The port to run the SFTP server on, expressed in Twisted's "strpo mini-language.
+  reporter:
+    type: string
+    default: local
+    description: The reporter used when generating OOPS reports.
\ No newline at end of file
diff --git a/charm/txpkgupload/layer.yaml b/charm/txpkgupload/layer.yaml
new file mode 100644
index 0000000..f959e97
--- /dev/null
+++ b/charm/txpkgupload/layer.yaml
@@ -0,0 +1,17 @@
+includes:
+  - layer:basic
+  - layer:ols
+  - interface:upload-queue-processor
+repo: https://git.launchpad.net/txpkgupload
+options:
+  apt:
+    packages:
+      - nodejs
+      - python3-convoy
+      - launchpad-soyuz-dependencies
+  ols:
+    service_name: txpkgupload
+    user: txpkgupload
+    tarball_payload: true
+    symlink_switch_payload: true
+    python_bin: /usr/bin/python3
diff --git a/charm/txpkgupload/metadata.yaml b/charm/txpkgupload/metadata.yaml
new file mode 100644
index 0000000..d6fffec
--- /dev/null
+++ b/charm/txpkgupload/metadata.yaml
@@ -0,0 +1,19 @@
+name: txpkgupload
+summary: txpkgupload
+maintainer: Launchpad Developers <launchpad-dev@xxxxxxxxxxxxxxxxxxx>
+description: |
+  Package upload server used by Launchpad.
+
+  Listens to FTP and SFTP requests, and processes uploads files into a
+  temporary directory
+tags:
+  # Replace "misc" with one or more whitelisted tags from this list:
+  # https://jujucharms.com/docs/stable/authors-charm-metadata
+  - misc
+series:
+  - focal
+subordinate: true
+requires:
+  upload-queue-processor:
+    interface: upload-queue-processor
+    scope: container
diff --git a/charm/txpkgupload/reactive/txpkgupload.py b/charm/txpkgupload/reactive/txpkgupload.py
new file mode 100644
index 0000000..0e0b619
--- /dev/null
+++ b/charm/txpkgupload/reactive/txpkgupload.py
@@ -0,0 +1,94 @@
+import os.path
+import subprocess
+from typing import Dict
+
+from charmhelpers.core import hookenv, templating
+from charms.reactive import (
+    clear_flag,
+    endpoint_from_flag,
+    set_flag,
+    when,
+    when_all,
+    when_not,
+    when_not_all,
+)
+from ols import base
+
+
+def configure_txpkgupload(config: Dict):
+    hookenv.log("Rendering txpkgupload_conf.yaml.")
+    templating.render(
+        "txpkgupload_config.yaml.j2",
+        config["config_dir"],
+        config,
+        owner="root",
+        group=base.user(),
+        perms=0o444,
+    )
+    hookenv.log("Rendering txpkgupload.service")
+    templating.render(
+        "txpkgupload.service.j2",
+        "/lib/systemd/system/txpkgupload.service",
+        config
+    )
+    # Socket activation to replace authbind in systemd
+    hookenv.log("Rendering txpkgupload.socket")
+    templating.render(
+        "txpkgupload.socket.j2",
+        "/lib/systemd/system/txpkgupload.socket",
+        config
+    )
+
+
+def get_config():
+    config = hookenv.config()
+    config.update(
+        {
+            "base_dir": base.base_dir(),
+            "code_dir": base.code_dir(),
+            "logs_dir": base.logs_dir(),
+            "user": base.user()
+        }
+    )
+
+    # Fetch data received from the parent upload queue processor service
+    uploader_relation = endpoint_from_flag(
+        "upload-queue-processor.configured"
+    )
+    config["fsroot"] = uploader_relation.fsroot
+    config["queue_processor_name"] = uploader_relation.processor_name
+
+    # Set directory for txpkgupload.yaml config file
+    config["config_dir"] = os.path.join(
+        base.code_dir(),
+        "production-configs",
+        "txpkgupload/txpkgupload.yaml",
+    )
+    config["pid_file"] = config["queue_processor_name"] + "_txpkgupload.pid"
+    return config
+
+
+@when_all(
+    "upload-queue-processor.configured",
+)
+@when_not("service.configured")
+def configure():
+    config = get_config()
+    configure_txpkgupload(config)
+
+    hookenv.log("Starting txpkgupload")
+    subprocess.run(["systemctl", "daemon-reload"])
+    set_flag("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+    hookenv.status_set("active", "Ready")
+
+
+@when_not_all(
+    "upload-queue-processor.configured",
+)
+@when("service.configured")
+def deconfigure():
+    clear_flag("service.configured")
diff --git a/charm/txpkgupload/templates/txpkgupload.service.j2 b/charm/txpkgupload/templates/txpkgupload.service.j2
new file mode 100644
index 0000000..590596c
--- /dev/null
+++ b/charm/txpkgupload/templates/txpkgupload.service.j2
@@ -0,0 +1,17 @@
+[Unit]
+Description=TX (Twisted) package upload helper
+After=network.target
+ConditionPathExists=!{{ code_dir }}/maintenance.txt
+
+[Service]
+User={{ user }}
+Group={{ user }}
+WorkingDirectory={{ code_dir }}
+Environment={{ queue_processor_name }}
+ExecStart={{ code_dir }}/bin/twistd --logfile={{ logs_dir }}/txpkgupload.log --pidfile={{ pid_file }} --nodaemon --umask=0 --config-file {{ config_dir }}
+ExecReload=/bin/kill -USR1 {{ pid_file }}
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target
diff --git a/charm/txpkgupload/templates/txpkgupload.socket.j2 b/charm/txpkgupload/templates/txpkgupload.socket.j2
new file mode 100644
index 0000000..017a978
--- /dev/null
+++ b/charm/txpkgupload/templates/txpkgupload.socket.j2
@@ -0,0 +1,8 @@
+[Unit]
+Description=Twisted package uploader service
+
+[Socket]
+ListenStream=/run/txpkgupload.socket
+
+[Install]
+WantedBy=sockets.target
\ No newline at end of file
diff --git a/charm/txpkgupload/templates/txpkgupload_config.yaml.j2 b/charm/txpkgupload/templates/txpkgupload_config.yaml.j2
new file mode 100644
index 0000000..ba9d079
--- /dev/null
+++ b/charm/txpkgupload/templates/txpkgupload_config.yaml.j2
@@ -0,0 +1,18 @@
+ftp:
+  port: {{ ftp_port }}
+
+sftp:
+  authentication_endpoint: {{ sftp_authentication_endpoint }}
+  host_key_private: {{ sftp_host_key_private }}
+  host_key_public: {{ sftp_host_key_public }}
+  port: {{ sftp_port }}
+
+oops:
+  directory: {{ logs_dir }}
+  reporter: {{ reporter }}
+
+access_log: {{ logs_dir }}/txpkgupload-access.log
+
+fsroot: {{ fsroot }}
+
+temp_dir: {{ base_dir }}/tmp-incoming
\ No newline at end of file
diff --git a/charm/txpkgupload/txpkgupload_ubuntu-20.04-amd64.charm b/charm/txpkgupload/txpkgupload_ubuntu-20.04-amd64.charm
new file mode 100644
index 0000000..5292ba1
Binary files /dev/null and b/charm/txpkgupload/txpkgupload_ubuntu-20.04-amd64.charm differ