RayServe application via ingress unreachable

Ray Libraries (Data, Train, Tune, Serve)
1

I have a RayCluster Helm Chart deployed on Azure Kubernetes Cluster. The cluster and dashboard is behind an Azure Application Gateway and NGINX Ingress Controller.
The dashboard is accessible but the RayServe applications are not reachable (404 not found) via ingress.

Upon using the port forward, I can reach the application.

Ingress

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /$1
    nginx.ingress.kubernetes.io/server-snippet: |
      underscores_in_headers on;
      ignore_invalid_headers on;
    nginx.ingress.kubernetes.io/use-regex: 'true'
  labels:
    app.kubernetes.io/instance: ray-cluster
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: kuberay
    argocd.argoproj.io/instance: ray-cluster
    helm.sh/chart: ray-cluster-1.0.0-rc.1
  name: ray-cluster-kuberay
spec:
  ingressClassName: nginx
  rules:
    - host: example.com
      http:
        paths:
          - backend:
              service:
                name: ray-cluster-kuberay-head-svc
                port:
                  number: 8265
            path: /raycluster-ingress/(.*)
            pathType: Prefix
    - host: example.com
      http:
        paths:
          - backend:
              service:
                name: ray-cluster-kuberay-head-svc
                port:
                  number: 6379
            path: /raycluster-ingress/(.*)
            pathType: Prefix
    - host: example.com
      http:
        paths:
          - backend:
              service:
                name: ray-cluster-kuberay-head-svc
                port:
                  number: 10001
            path: /raycluster-ingress/(.*)
            pathType: Prefix
    - host: example.com
      http:
        paths:
          - backend:
              service:
                name: ray-cluster-kuberay-head-svc
                port:
                  number: 8000
            path: /raycluster-ingress/(.*)
            pathType: Prefix
    - host: example.com
      http:
        paths:
          - backend:
              service:
                name: ray-cluster-kuberay-head-svc
                port:
                  number: 8080
            path: /raycluster-ingress/(.*)
            pathType: Prefix
  tls:
    - hosts:
        - example.com
      secretName: dev-eu-cert

Service

apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/created-by: kuberay-operator
    app.kubernetes.io/name: kuberay
    ray.io/cluster: ray-cluster-kuberay
    ray.io/identifier: ray-cluster-kuberay-head
    ray.io/node-type: head
  name: ray-cluster-kuberay-head-svc
  ownerReferences:
    - apiVersion: ray.io/v1
      blockOwnerDeletion: true
      controller: true
      kind: RayCluster
      name: ray-cluster-kuberay
spec:
  clusterIP: 10.68.98.168
  clusterIPs:
    - 10.68.98.168
  internalTrafficPolicy: Cluster
  ipFamilies:
    - IPv4
  ipFamilyPolicy: SingleStack
  ports:
    - appProtocol: tcp
      name: metrics
      port: 8080
      protocol: TCP
      targetPort: 8080
    - appProtocol: tcp
      name: serve
      port: 8000
      protocol: TCP
      targetPort: 8000
  selector:
    app.kubernetes.io/created-by: kuberay-operator
    app.kubernetes.io/name: kuberay
    ray.io/cluster: ray-cluster-kuberay
    ray.io/identifier: ray-cluster-kuberay-head
    ray.io/node-type: head
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}

RayCluster

apiVersion: ray.io/v1
kind: RayCluster
metadata:
  labels:
    app.kubernetes.io/instance: ray-cluster
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: kuberay
    argocd.argoproj.io/instance: ray-cluster
    helm.sh/chart: ray-cluster-1.0.0-rc.1
  name: ray-cluster-kuberay
spec:
  headGroupSpec:
    rayStartParams:
      dashboard-host: 0.0.0.0
    serviceType: ClusterIP
    template:
      metadata:
        annotations: {}
        labels:
          app.kubernetes.io/instance: ray-cluster
          app.kubernetes.io/managed-by: Helm
          app.kubernetes.io/name: kuberay
          helm.sh/chart: ray-cluster-1.0.0-rc.1
      spec:
        containers:
          - env: []
            image: 'rayproject/ray:2.7.0'
            imagePullPolicy: IfNotPresent
            name: ray-head
            ports:
              - containerPort: 8000
                name: serve
                protocol: TCP
            resources:
              limits:
                cpu: '4'
                memory: 16G
              requests:
                cpu: '4'
                memory: 16G
            securityContext:
              allowPrivilegeEscalation: false
              capabilities:
                drop:
                  - ALL
              privileged: false
              readOnlyRootFilesystem: false
              runAsGroup: 100000
              runAsNonRoot: true
              runAsUser: 100000
            volumeMounts:
              - mountPath: /tmp/ray
                name: log-volume
        imagePullSecrets: []
        nodeSelector: {}
        volumes:
          - emptyDir: {}
            name: log-volume
  workerGroupSpecs:
    - groupName: workergroup
      maxReplicas: 2147483647
      minReplicas: 0
      rayStartParams: {}
      replicas: 1
      template:
        metadata:
          annotations: {}
          labels:
            app.kubernetes.io/instance: ray-cluster
            app.kubernetes.io/managed-by: Helm
            app.kubernetes.io/name: kuberay
            helm.sh/chart: ray-cluster-1.0.0-rc.1
        spec:
          containers:
            - env: []
              image: 'rayproject/ray:2.7.0'
              imagePullPolicy: IfNotPresent
              name: ray-worker
              ports: null
              resources:
                limits:
                  cpu: '4'
                  memory: 16G
                requests:
                  cpu: '4'
                  memory: 16G
              securityContext:
                allowPrivilegeEscalation: false
                capabilities:
                  drop:
                    - ALL
                privileged: false
                readOnlyRootFilesystem: false
                runAsGroup: 100000
                runAsNonRoot: true
                runAsUser: 100000
              volumeMounts:
                - mountPath: /tmp/ray
                  name: log-volume
          imagePullSecrets: []
          nodeSelector: {}
          volumes:
            - emptyDir: {}
              name: log-volume