Serve multiple applications in one run with FastAPI

Ray Libraries (Data, Train, Tune, Serve)
1

Hi Team, I was following this link Deploy Multiple Applications — Ray 2.23.0 but additional with FastAPI & uvricorn to expose a api link.

my yaml file looks like below

proxy_location: EveryNode

http_options:
  host: 0.0.0.0
  port: 8000

grpc_options:
  port: 9000
  grpc_servicer_functions: []

logging_config:
  encoding: JSON
  log_level: INFO
  logs_dir: null
  enable_access_log: true

applications:
  - name: scenario_learning_app
    route_prefix: /api/next_phrase
    import_path: chatbots.scenarios_phrase_learning.scenarios_learning:get_app_scenario_learning
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: ScenarioLearningDeployment

  - name: translator_app
    route_prefix: /api/translate
    import_path: chatbots.translator:get_app_translator
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: TranslatorDeployment

  - name: caption_translation_app
    route_prefix: /api/process_image
    import_path: chatbots.img_to_text.caption_translation:get_app_image_handler
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: ImageHandlerDeployment

  - name: generated_audio_app
    route_prefix: /api/generate_audio/en
    import_path: chatbots.audio.generated_audio:get_app_audio_generator
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: AudioGeneratorDeployment

  - name: user_audio_app
    route_prefix: /api/evaluatePronunciation
    import_path: chatbots.audio.user_audio:get_app_audio_evaluator
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: AudioEvaluatorDeployment

  - name: sqllite_database_app
    route_prefix: /api/register
    import_path: chatbots.database.sqllite_database:get_app_user_management
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: UserManagementDeployment

  - name: phrase_learning_app
    route_prefix: /api/next_phrase1
    import_path: chatbots.phrase_learning.phrase_learning:get_app_dictionary_viewer
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: DictionaryViewerDeployment

  - name: english_quiz_app
    route_prefix: /api/start_quiz
    import_path: chatbots.quiz.english_quiz:get_app_quiz_handler
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: QuizHandlerDeployment

  - name: scenario_selection_app
    route_prefix: /api/chat
    import_path: chatbots.scenarios_chat.scenario_selection:get_app_chatbot
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: ChatbotDeployment

  - name: fetch_edict_app
    route_prefix: /api/edict
    import_path: chatbots.speaking_from_dict.fetch_edict:get_app_ec_dict
    runtime_env: {}
    deployments:
      - name: APIIngress
      - name: ECDictDeployment

And one of my application looks like this

import csv
from http.client import HTTPException
from typing import Optional

from fastapi import FastAPI
from pydantic import BaseModel

from ray import serve
from ray.serve.handle import DeploymentHandle

class ECDict:
    def __init__(self, csv_file_path):
        self.dictionary = self.load_dictionary(csv_file_path)

    def load_dictionary(self, csv_file_path):
        dictionary = {}
        with open(csv_file_path, mode='r', encoding='utf-8') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                word = row['word']
                dictionary[word] = {
                    'phonetic': row['phonetic'],
                    'definition': row['definition'],
                    'translation': row['translation']
                }
        return dictionary

    def lookup_word(self, word):
        return self.dictionary.get(word, None)


class LookupRequest(BaseModel):
    word: str

class WordDetail(BaseModel):  # Pydantic model to structure response data
    phonetic: Optional[str] = None
    definition: Optional[str] = None
    translation: Optional[str] = None

app = FastAPI()
csv_file_path = ('/home/resources/ecdict.csv')

@serve.deployment
@serve.ingress(app)
class APIIngress:
    def __init__(self, ec_dict_handle: DeploymentHandle) -> None:
        self.handle = ec_dict_handle

    @app.post("/api/edict", response_model=WordDetail)
    async def lookup_word(self, request: LookupRequest):
        result = await self.handle.lookup_word.remote(request.word)
        if result:
            return WordDetail(**result)
        else:
            raise HTTPException(status_code=404, detail="Word not found")

@serve.deployment(
    ray_actor_options={"num_gpus": 0},
    autoscaling_config={"min_replicas": 1, "max_replicas": 2},
)
class ECDictDeployment:
    def __init__(self):
        self.ec_dict = ECDict(csv_file_path=csv_file_path)

    def lookup_word(self, word: str):
        return self.ec_dict.lookup_word(word)

entrypoint = APIIngress.bind(ECDictDeployment.bind())

def get_app_ec_dict(config: dict):
    return entrypoint

if __name__ == "__main__":
    import ray
    ray.init(address="auto")
    serve.start()
    serve.run(get_app_ec_dict())
    # import uvicorn
    # uvicorn.run(app, host="0.0.0.0", port=8000)

Currently the serve status looks healthy and successfully deployed the applications. However when I was checking on swagger http://localhost:8800/api/edict

the result is {“detail”:“Not Found”}

Not sure if issue has been spotted before, looking forward to your reply thanks!