Integrate selkies-gstreamer to Jupyter and Coder (just like noVNC)

[ehfd]

Update:

We currently have this Coder configuration working self-hosted:

entrypoint.sh for Coder:

set -e
supervisord

Open main.tf

terraform {
  required_providers {
    coder = {
      source  = "coder/coder"
      version = "~> 0.6.17"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.18"
    }
  }
}

provider "coder" {
  feature_use_managed_variables = true
}

provider "kubernetes" {
  config_path = var.use_kubeconfig == true ? "~/.kube/config" : null
}

data "coder_workspace" "me" {}

data "coder_parameter" "cpu" {
  name    = "CPU (cores)"
  default = "2"
  icon    = "/icon/memory.svg"
  mutable = true
  option {
    name  = "2 Cores"
    value = "2"
  }
  option {
    name  = "4 Cores"
    value = "4"
  }
  option {
    name  = "6 Cores"
    value = "6"
  }
  option {
    name  = "8 Cores"
    value = "8"
  }
  option {
    name  = "16 Cores"
    value = "16"
  }
}

data "coder_parameter" "memory" {
  name    = "Memory (GB)"
  default = "2"
  icon    = "/icon/memory.svg"
  mutable = true
  option {
    name  = "2 GB"
    value = "2"
  }
  option {
    name  = "4 GB"
    value = "4"
  }
  option {
    name  = "6 GB"
    value = "6"
  }
  option {
    name  = "8 GB"
    value = "8"
  }
    option {
    name  = "16 GB"
    value = "16"
  }
  option {
    name  = "32 GB"
    value = "32"
  }
  option {
    name  = "64 GB"
    value = "64"
  }
}

data "coder_parameter" "password" {
  name         = "password"
  display_name = "Selkies Password"
  description  = "The Selkies password for authentication. User is ubuntu."
  icon         = "/emojis/1f511.png"
  mutable      = false
  option {
    name  = "mypasswd"
    value = "mypasswd"
  }
}

data "coder_parameter" "home_disk_size" {
  name         = "home_disk_size"
  display_name = "Home disk size"
  description  = "The size of the home disk in GB"
  default      = "100"
  type         = "number"
  icon         = "/emojis/1f4be.png"
  mutable      = false
  validation {
    min = 1
    max = 99999
  }
}



variable "use_kubeconfig" {
  type        = bool
  description = "Use host kubeconfig? (true/false)"
  default     = false
}

variable "namespace" {
  type        = string
  description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces)"
  default     = "coder"
}


resource "kubernetes_pod" "main" {
  count = data.coder_workspace.me.start_count
  metadata {
    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
    namespace = var.namespace
    labels = {
      "app.kubernetes.io/name"     = "coder-workspace"
      "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
      "app.kubernetes.io/part-of"  = "coder"
      // Coder specific labels.
      "com.coder.resource"       = "true"
      "com.coder.workspace.id"   = data.coder_workspace.me.id
      "com.coder.workspace.name" = data.coder_workspace.me.name
      "com.coder.user.id"        = data.coder_workspace.me.owner_id
      "com.coder.user.username"  = data.coder_workspace.me.owner
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace.me.owner_email
    }
  }

  spec {

    container {
      name  = "dev"
      image = "ghcr.io/selkies-project/nvidia-glx-desktop:latest"
      image_pull_policy = "Always"
      command           = ["sh", "-c", coder_agent.main.init_script]
      # security_context {
      #   run_as_user = "1000"
      #   privileged = true
      # }

      env {
        name  = "TZ"
        value = "UTC"
      }
      env {
        name  = "DISPLAY_SIZEW"
        value = "1920"
      }
      env {
        name  = "DISPLAY_SIZEH"
        value = "1080"
      }
      env {
        name  = "DISPLAY_REFRESH"
        value = "60"
      }
      env {
        name  = "DISPLAY_DPI"
        value = "96"
      }
      env {
        name  = "DISPLAY_CDEPTH"
        value = "24"
      }
      env {
        name  = "VIDEO_PORT"
        value = "DFP"
      }
      env {
        name  = "PASSWD"
        value = "${data.coder_parameter.password.value}"
      }
      env {
        name  = "SELKIES_ENCODER"
        value = "nvh264enc"
      }
      env {
        name  = "SELKIES_ENABLE_RESIZE"
        value = "false"
      }
      env {
        name  = "SELKIES_VIDEO_BITRATE"
        value = "8000"
      }
      env {
        name  = "SELKIES_FRAMERATE"
        value = "60"
      }
      env {
        name  = "SELKIES_AUDIO_BITRATE"
        value = "128000"
      }
      env {
        name  = "SELKIES_ENABLE_BASIC_AUTH"
        value = "true"
      }
      env {
        name  = "SELKIES_BASIC_AUTH_PASSWORD"
        value = "${data.coder_parameter.password.value}"
      }
      env {
        name  = "SELKIES_TURN_REST_URI"
        value = "http://turn-rest.nrp-nautilus.io"
      }
      env {
        name  = "SELKIES_TURN_PROTOCOL"
        value = "tcp"
      }
      env {
        name  = "SELKIES_TURN_TLS"
        value = "false"
      }
      env {
        name  = "CODER_AGENT_TOKEN"
        value = coder_agent.main.token
      }

      stdin = true
      tty   = true

      port {
        name           = "http"
        container_port = 8080
        protocol       = "TCP"
      }

      resources {
        limits = {
          "cpu"    = "${data.coder_parameter.cpu.value}"
          "memory" = "${data.coder_parameter.memory.value}Gi"
          "nvidia.com/gpu" = 1
        }
        requests = {
          "cpu"    = "${data.coder_parameter.cpu.value}"
          "memory" = "${data.coder_parameter.memory.value}Gi"
          "nvidia.com/gpu" = 1
        }
      }

      volume_mount {
        mount_path = "/home/ubuntu/persistent"
        name       = "home"
        read_only  = false
        sub_path   = "home"
      }

      volume_mount {
        mount_path = "/dev/shm"
        name       = "dshm"
      }
    }

    dns_policy = "None"
    dns_config {
      nameservers = ["8.8.8.8", "8.8.4.4"]
    }

    volume {
      name = "dshm"
      empty_dir {}
    }

    volume {
      name = "home"
      persistent_volume_claim {
        claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
        read_only  = false
      }
    }

    affinity {
      node_affinity {
        required_during_scheduling_ignored_during_execution {
          node_selector_term {
            match_expressions {
              key      = "topology.kubernetes.io/zone"
              operator = "NotIn"
              values   = ["myzone"]
            }
          }
        }
      }
    }
  }
}

resource "coder_agent" "main" {
  os                     = "linux"
  arch                   = "amd64"
  login_before_ready     = false
  startup_script_timeout = 180
  startup_script         = <<-EOT
    set -e

    # install and start code-server
    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server --version 4.8.3
    /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 &

    echo "Initializing Supervisor..."
    nohup supervisord
  EOT
}

resource "coder_app" "code-server" {
  agent_id     = coder_agent.main.id
  slug         = "code-server"
  display_name = "code-server"
  icon         = "/icon/code.svg"
  url          = "http://localhost:13337?folder=/home/coder"
  subdomain    = false
  share        = "owner"

  healthcheck {
    url       = "http://localhost:13337/healthz"
    interval  = 3
    threshold = 10
  }
}

resource "kubernetes_persistent_volume_claim" "home" {
  metadata {
    name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}-home"
    namespace = var.namespace
    labels = {
      "app.kubernetes.io/name"     = "coder-pvc"
      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
      "app.kubernetes.io/part-of"  = "coder"
      // Coder specific labels.
      "com.coder.resource"       = "true"
      "com.coder.workspace.id"   = data.coder_workspace.me.id
      "com.coder.workspace.name" = data.coder_workspace.me.name
      "com.coder.user.id"        = data.coder_workspace.me.owner_id
      "com.coder.user.username"  = data.coder_workspace.me.owner
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace.me.owner_email
    }
  }
  wait_until_bound = false
  spec {
    access_modes = ["ReadWriteOnce"]
    storage_class_name = "rook-ceph-block"
    resources {
      requests = {
        storage = "${data.coder_parameter.home_disk_size.value}Gi"
      }
    }
  }
}

resource "coder_app" "selkies" {
  agent_id      = coder_agent.main.id
  slug          = "selkies"  
  display_name  = "Selkies"
  icon          = "/emojis/1f3ae.png"
  url           = "http://localhost:8080"
  subdomain     = true
  share         = "owner"
}

---

Self-explanatory. Just like VS Code Server and noVNC, a button click in Jupyter should lead to a window with Selkies.
This will help greatly in robotics, simulations, and other kinds of research.

External contribution dearly expected.

It can be a separate project, a PR, or any other form of contribution.
Integration with Jupyter Docker containers should also be possible.

Creating a template for Coder would also be of interest.

[Inrixia]

Messing around trying to get this working with coder.

Can access the web endpoint fine but webrtc fails.

When using coder port-forward to open needed ports for a direct connection webrtc also fails as the signaling handshake specifies the private host ip of the container which is not resolvable eg 10.x.x.x vs 127.0.0.1.

Ideally there would be a way to manually specify the ports/ip used to get around this, though I'm not sure how tenable that is.

I haven't tested using a turn server because if I have to fall back to that I'll just end up going with something like KasmVNC which works seamlessly over a single port.

[Inrixia]

This looks super promising. Will check it out tomorrow, thanks!

[Inrixia]

Hey, have messed around a bit with this. Had to switch to using the egl container as I am not using a gpu for testing.

While this boots and works fine. It still requires a turn server which does not work due to firewall restrictions.

Plus I cannot justify needing to run a seperate turn server.

How difficult would it be to be able to pin ports needed so that a direct connection could be established using coder port-forward

[ehfd]

https://github.com/selkies-project/docker-nvidia-egl-desktop#running-with-kubernetes

Maybe the internal TURN server option could work here.

[Inrixia]

That might work since can limit it to only two ports. Will give it a try and see!

Though actually may run into the issue earlier where the internal ip of the pod doesn't match the forwarded (127.0.0.1). Will update how it goes

[Inrixia]

Yea so when doing ice negotiation it is advertising the internal pod ips (10.x.x.x) so ice still fails as that address is not resolvable. It has to advertise something like 127.0.0.1 or local host etc so that the client attempts to connect via the port forward.

I ran into this issue when originally testing as mentioned above.

Having a option to override what ip is advertised would probably fix this.

[Inrixia]

Actually I don't seem to be able to get it to use the internal turn server.

Checking the logs its still advertising defaults

[Inrixia]

Wait I'm being dumb. Was missing the turn host env var, it's further down the docs and didn't notice it.

[Inrixia]

Hmm, now I'm getting proper advertising but even though the internal server advertises 65535 as a option it is still listening on the old ports and not 65535