Hyperpipe & Piped

Introduction

Cet article à pour but d’illustrer pas loin de deux mois de lutte acharnés afin de mettre en place une solution qui devait me permettre d’utiliser Youtube; mais sans la publicité.

Prérequis

Il faut avoir lu mes articles sur Traefik (lui est les autres) :)

Fichiers requis

docker-compose.yml

[Fichier]

version: "3.0"

#
# updated: 2023-06-25
# stack:   hyperpipe
#

x-logging: &x-logging
  logging:
    driver: loki
    options:
      loki-url: "http://loki:3100/loki/api/v1/push"
      loki-retries: "5"
      loki-batch-size: "400"

x-environment: &x-environment
    TZ: "Europe/Paris"
    PUID: 1000
    PGID: 1000

x-common: &x-common
  <<: *x-logging
  restart: "no"
  stop_grace_period: 5s
  stdin_open: true
  tty: true
  privileged: false
  security_opt:
    - no-new-privileges=true
  cap_drop:
    - ALL
  cap_add:
    - KILL
  dns:
    - 1.1.1.1
    - 8.8.8.8
    - 1.0.0.1
    - 8.8.4.4
  ipc: "shareable"
  extra_hosts:
    - "template.home:192.168.0.0"
  environment:
    *x-environment
  user: 1000:1000
  labels:
    com.centurylinklabs.watchtower.enable: true
    logging: "promtail"
    com.stack.name: "common"
    com.stack.service.name: "common"
  devices:
    - /dev/kmsg:/dev/kmsg
  deploy:
    resources:
      limits:
        cpus: "0.50"
        memory: 256M
  ulimits:
    nproc: 65535
    nofile:
      soft: 20000
      hard: 40000
  tmpfs:
    - /tmp:rw,noexec,nosuid,size=64k
  sysctls:
    net.core.somaxconn: 1024
    net.ipv4.tcp_syncookies: 0
x-volume-timezone: &x-volume-timezone "/etc/timezone:/etc/timezone:ro"
x-volume-localtime: &x-volume-localtime "/etc/localtime:/etc/localtime:ro"
x-volume-docker-socket: &x-volume-docker-socket "/var/run/docker.sock:/var/run/docker.sock:rw"
x-volume-cgroups: &x-volume-cgroups "/proc/cgroups:/cgroup:rw"
x-volume-ssl: &x-volume-ssl "/opt/docker/ssl:/ssl:ro"

services:
  piped-db:
    <<: *x-common
    user: 0:0
    cap_add:
      - DAC_OVERRIDE
      - CHOWN
      - FOWNER
      - FSETID
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
      - MKNOD
    container_name: piped-db
    hostname: piped-db
    image: postgres:latest
    restart: unless-stopped
    ports:
      - "5433:5432"
    expose:
      - "5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-q", "-d", "piped", "-U", "root"]
      timeout: 45s
      interval: 10s
      retries: 10
    environment:
      <<: *x-environment
      POSTGRES_DB: piped
      POSTGRES_USER: [username]
      POSTGRES_PASSWORD: [password]
    labels:
      com.stack.name: "piped"
      com.stack.service.name: "db"
    deploy:
      resources:
        limits:
          memory: 512M
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
      - /opt/docker/hyperpipe/datas/db:/var/lib/postgresql/data:rw
  piped-proxy:
    <<: *x-common
    read_only: true
    container_name: piped-proxy
    hostname: piped-proxy
    image: 1337kavin/piped-proxy:latest
    restart: unless-stopped
    environment:
      <<: *x-environment
      UDS: 1
    labels:
      com.stack.name: "piped"
      com.stack.service.name: "proxy"
    deploy:
      resources:
        limits:
          memory: 512M
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
      - /opt/docker/hyperpipe/datas/piped-proxy:/app/socket:rw
  piped-back:
    <<: *x-common
    user: 0:0
    cap_add:
      - DAC_OVERRIDE
      - CHOWN
      - FOWNER
      - FSETID
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
      - MKNOD
    container_name: piped-back
    hostname: piped-back
    #image: 1337kavin/piped:latest
    image: zogg/piped:latest
    restart: unless-stopped
    ports:
      - "8046:8080"
    expose:
      - "8080"
    depends_on:
      - piped-db
    healthcheck:
      test: curl -f http://localhost:8080/ || exit 1
    environment:
      <<: *x-environment
      DSN: ""
    labels:
      com.stack.name: "piped"
      com.stack.service.name: "back"
    deploy:
      resources:
        limits:
          memory: 2G
    tmpfs:
      - /tmp:rw,exec,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
      - /opt/docker/hyperpipe/conf/config.properties:/app/config.properties:ro
  piped-front:
    <<: *x-common
    user: 0:0
    cap_add:
      - DAC_OVERRIDE
      - CHOWN
      - FOWNER
      - FSETID
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
      - MKNOD
    container_name: piped-front
    hostname: piped-front
    image: 1337kavin/piped-frontend:latest
    restart: unless-stopped
    ports:
      - "8047:80"
    expose:
      - "80"
    depends_on:
      - piped-back
    healthcheck:
      test: wget --no-verbose --tries=1 --spider http://localhost:80
    entrypoint: ash -c 'sed -i s/pipedapi.kavin.rocks/pipedapi.domain.com/g /usr/share/nginx/html/assets/* && /docker-entrypoint.sh && nginx -g "daemon off;"'
    labels:
      com.stack.name: "piped"
      com.stack.service.name: "front"
    deploy:
      resources:
        limits:
          memory: 1G
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
  hyperpipe-nginx:
    <<: *x-common
    user: 0:0
    cap_add:
      - DAC_OVERRIDE
      - CHOWN
      - FOWNER
      - FSETID
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
      - MKNOD
    container_name: hyperpipe-nginx
    hostname: hyperpipe-nginx
    image: nginx:latest
    restart: unless-stopped
    depends_on:
      - piped-back
      - piped-front
      - piped-proxy
    ports:
      - "8045:80"
    expose:
      - "80"
    healthcheck:
      test: curl -f http://localhost:80/ || exit 1
    labels:
      com.stack.name: "hyperpipe"
      com.stack.service.name: "nginx"
    deploy:
      resources:
        limits:
          memory: 512M
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
      - /opt/docker/hyperpipe/conf/nginx.conf:/etc/nginx/nginx.conf:ro
      - /opt/docker/hyperpipe/conf/pipedapi.conf:/etc/nginx/conf.d/pipedapi.conf:ro
      - /opt/docker/hyperpipe/conf/pipedproxy.conf:/etc/nginx/conf.d/pipedproxy.conf:ro
      - /opt/docker/hyperpipe/conf/pipedfrontend.conf:/etc/nginx/conf.d/pipedfrontend.conf:ro
      - /opt/docker/hyperpipe/conf/ytproxy.conf:/etc/nginx/snippets/ytproxy.conf:ro
      - /opt/docker/hyperpipe/datas/piped-proxy:/var/run/ytproxy:rw
  hyperpipe-back:
    <<: *x-common
    container_name: hyperpipe-back
    hostname: hyperpipe-back
    image: codeberg.org/hyperpipe/hyperpipe-backend:latest
    restart: unless-stopped
    depends_on:
      - hyperpipe-nginx
    ports:
      - "3771:3000"
    expose:
      - "3000"
    environment:
      <<: *x-environment
      HYP_PROXY: "hyperpipe-proxy.onrender.com"
    labels:
      com.stack.name: "hyperpipe"
      com.stack.service.name: "back"
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime
  hyperpipe-front:
    <<: *x-common
    user: 0:0
    cap_add:
      - DAC_OVERRIDE
      - CHOWN
      - FOWNER
      - FSETID
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
      - MKNOD
    container_name: hyperpipe-front
    hostname: hyperpipe-front
    image: codeberg.org/hyperpipe/hyperpipe:latest
    restart: unless-stopped
    depends_on:
      - hyperpipe-back
    ports:
      - "8745:80"
    expose:
      - "80"
    healthcheck:
      test: wget --no-verbose --tries=1 --spider http://localhost
    entrypoint: sh -c 'find /usr/share/nginx/html -type f -exec sed -i s/pipedapi.kavin.rocks/pipedapi.domain.com/g {} \; -exec sed -i s/hyperpipeapi.onrender.com/hyperpipeapi.domain.com/g {} \; && /docker-entrypoint.sh && nginx -g "daemon off;"'
    labels:
      com.stack.name: "hyperpipe"
      com.stack.service.name: "front"
    deploy:
      resources:
        limits:
          memory: 512M
    tmpfs:
      - /tmp:rw,noexec,nosuid,size=512M
    volumes:
      - *x-volume-timezone
      - *x-volume-localtime

config.properties

[Fichier]

# The port to Listen on.
PORT: 8080

# The number of workers to use for the server
HTTP_WORKERS: 8

# Proxy
PROXY_PART: https://pipedproxy.domain.com

# Outgoing HTTP Proxy - eg:  127.0.0.1: 8118
#HTTP_PROXY:  127.0.0.1: 8118

# Captcha Parameters
#CAPTCHA_BASE_URL:  https: //api.capmonster.cloud/
#CAPTCHA_API_KEY:  INSERT_HERE

# Public API URL
API_URL: https://pipedapi.domain.com

# Public Frontend URL
FRONTEND_URL: https://videos.domain.com

# Enable haveibeenpwned compromised password API
COMPROMISED_PASSWORD_CHECK: true

# Disable Registration
DISABLE_REGISTRATION: false

# Feed Retention Time in Days
FEED_RETENTION: 30

# Sentry DSN
SENTRY_DSN:""

# Hibernate properties
hibernate.connection.url: jdbc:postgresql://[adresse ip]:5433/piped
hibernate.connection.driver_class: org.postgresql.Driver
hibernate.dialect: org.hibernate.dialect.PostgreSQLDialect
hibernate.connection.username: [nom utilisateur]
hibernate.connection.password: [mot de passe]

pipedfrontend.conf

[Fichier]

server {
    listen 80;
    server_name piped.domain.com;

    set $backend "http://[adresse ip]:8047";

    location / {
        proxy_pass $backend;
        proxy_http_version 1.1;
        proxy_set_header Connection "keep-alive";
    }
}

pipedapi.conf

[Fichier]

proxy_cache_path /tmp/pipedapi_cache levels=1:2 keys_zone=pipedapi:4m max_size=2g inactive=60m use_temp_path=off;

server {
    listen 80;
    server_name pipedapi.domain.com;

    set $backend "http://[adresse ip]:8046";

    location / {
        proxy_cache pipedapi;
        proxy_pass $backend;
        proxy_http_version 1.1;
        proxy_set_header Connection "keep-alive";
    }
}

pipedproxy.conf

[Fichier]

server {
    listen 80;
    server_name pipedproxy.domain.com;

    location ~ (/videoplayback|/api/v4/|/api/manifest/) {
        include snippets/ytproxy.conf;
        add_header Cache-Control private always;
    }

    location / {
        include snippets/ytproxy.conf;
        add_header Cache-Control "public, max-age=604800";
    }
}

ytproxy.conf

[Fichier]

proxy_buffering on;
proxy_buffers 1024 16k;
proxy_set_header X-Forwarded-For "";
proxy_set_header CF-Connecting-IP "";
proxy_hide_header "alt-svc";
sendfile on;
sendfile_max_chunk 512k;
tcp_nopush on;
aio threads=default;
aio_write on;
directio 16m;
proxy_hide_header Cache-Control;
proxy_hide_header etag;
proxy_http_version 1.1;
proxy_set_header Connection keep-alive;
proxy_max_temp_file_size 32m;
access_log off;
proxy_pass http://unix:/var/run/ytproxy/actix.sock;

nginx.conf

[Fichier]

user root;
worker_processes auto;

error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;

thread_pool default threads=512 max_queue=65536;

events {
    worker_connections 4096;
    multi_accept on;
    use epoll;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    server_names_hash_bucket_size 128;

    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
    '$status $body_bytes_sent "$http_referer" '
    '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log /var/log/nginx/access.log main;
    access_log off;

    sendfile on;
    tcp_nodelay on;

    keepalive_timeout 65;

    resolver [adresse ip] ipv6=off valid=10s;

    server_tokens off;
    tcp_nopush on;
    types_hash_max_size 2048;
    client_body_buffer_size 64K;
    client_header_buffer_size 64k;
    client_max_body_size 128k;
    large_client_header_buffers 8 16k;
    keepalive_requests 100000;
    send_timeout 30;
    client_body_timeout 30;
    client_header_timeout 30;
    reset_timedout_connection on;
    open_file_cache max=2000 inactive=20s;
    open_file_cache_valid 60s;
    open_file_cache_min_uses 5;
    open_file_cache_errors off;
    add_header X-XSS-Protection "1; mode=block";
    add_header X-Frame-Options "SAMEORIGIN";
    add_header X-Content-Type-Options nosniff;
    add_header Strict-Transport-Security "max-age=63072000" always;
    gzip on;
    gzip_static on;
    gzip_min_length 1024;
    gzip_comp_level 6;
    gzip_http_version 1.1;
    gzip_vary on;
    gzip_disable msie6;
    gzip_disable "MSIE [1-6]\.";
    gzip_proxied any;
    gzip_buffers 16 8k;
    gzip_types
        text/plain
        text/css
        text/xml
        text/javascript
        text/x-component
        application/x-javascript
        application/xml
        application/javascript
        application/json
        application/xml+rss
        application/rss+xml
        application/atom+xml
        font/truetype
        font/opentype
        application/vnd.ms-fontobject
        image/svg+xml
        application/geo+json
        application/ld+json
        application/manifest+json
        application/rdf+xml
        application/wasm
        application/x-web-app-manifest+json
        application/xhtml+xml
        font/eot
        font/otf
        font/ttf
        image/bmp
        text/cache-manifest
        text/calendar
        text/markdown
        text/vcard
        text/vnd.rim.location.xloc
        text/vtt
        text/x-cross-domain-policy
        application/x-font-ttf
        image/x-icon;

    include /etc/nginx/conf.d/*.conf;
}

platform.sh

[Fichier]

#!/bin/bash

# Used in Docker build to set platform dependent variables

case $TARGETARCH in

    "amd64")
	echo "x86_64-unknown-linux-gnu" > /.platform
	echo "" > /.compiler
	;;
    "arm64")
	echo "aarch64-unknown-linux-gnu" > /.platform
	echo "gcc-aarch64-linux-gnu" > /.compiler
	;;
    "arm")
	echo "armv7-unknown-linux-gnueabihf" > /.platform
	echo "gcc-arm-linux-gnueabihf" > /.compiler
	;;
esac

build.sh

[Fichier]

#!/bin/bash
# 2023-06-25

clear
cd "$(dirname "$0")" || exit 1

IMAGE_BASE=zogg/piped
IMAGE_NAME_LATEST=${IMAGE_BASE}:latest

export DOCKER_CLI_EXPERIMENTAL=enabled
docker run --privileged --rm tonistiigi/binfmt --install all

export DOCKER_DEFAULT_PLATFORM=linux/amd64
docker buildx build --pull \
    --platform=linux/amd64 \
    --output=type=docker \
    --build-arg TZ=Europe/Paris \
    --build-arg CONCURRENCY=$(nproc) \
    -t "${IMAGE_NAME_LATEST}" \
    . 2>&1 | tee build.log

exit 0

Préparation

Il est nécessaire de construire une image modifée de Piped pour que l’ensemble puisse fonctionner.

Sans cette modification, le conteneur piped-back refusera se lancer correctement avec l’erreur suivante :

Exception in thread “main” java.lang.IllegalArgumentException: DSN is required. Use empty string to disable SDK.

Pour ce faire, dans un shell, vous tappez :

git clone https://github.com/TeamPiped/Piped-Backend

Dans ce répertoire, vous copier les fichiers suivants :

• plateform.sh
• build.sh

Ensuite, vous vous rendez dans le répertoire src/main/java/me/kavin/piped.

Vous éditez le fichier Main.java et vous commentez le code Sentry.init somme suit :

public class Main {

    public static void main(String[] args) throws Exception {

        NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor());
        YoutubeStreamExtractor.forceFetchAndroidClient(true);
        YoutubeStreamExtractor.forceFetchIosClient(true);

        /*
        Sentry.init(options -> {
            options.setDsn(Constants.SENTRY_DSN);
            options.setRelease(Constants.VERSION);
            options.addIgnoredExceptionForType(ErrorResponse.class);
            options.setTracesSampleRate(0.1);
        });
        */

        Injector.useSpecializer();

Puis vous revenez à la racine du projet; là ou vous avez cloner le dépôt Git.

Et vous lancez la compilation de l’image Docker :

sudo bash build.sh

Et c’est cette image qui sera utilisée dans cette partie de la stack :

  piped-back:
    ...
    #image: 1337kavin/piped:latest
    image: zogg/piped:latest

Mise en place

Je partirai du principe que vous utilisez Portainer pour gérer vos stack Docker.

Il vous faudra créer les répertoires suivants (ou adapter au besoin) :

• /opt/docker/hyperpipe/conf/
• /opt/docker/hyperpipe/datas/db/
• /opt/docker/hyperpipe/datas/piped-proxy/

Précisions

L’utilisation de ce genre d’outil requiert une bonne bande passante (à la fois montante et descendante).

Conclusion

Avec cette solution vous disposer d’un moyen efficace de visionner des vidéos Youtube ou d’écouter de la musique sur Youtube Music; et le tout sans publicité !