How can I use SSL for deployment? I need to set up SSL certificates and use TLS.
I can’t even find the place where it is possible to turn on it.
import requests
import torch
import json
from transformers import RobertaModel, RobertaConfig, RobertaTokenizer
import os
import numpy as np
from torch.utils.data import DataLoader, Dataset, SequentialSampler, TensorDataset
from ray import serve
import ray
from fastapi import FastAPI
from pathlib import Path
# 1: Define a FastAPI app and wrap it in a deployment with a route handler.
app = FastAPI()
def load_model_local(config_model, device):
# build model
tokenizer = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_tokenizer.pt"), map_location=device)
config = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_config.pt"))
config.is_decoder = True
model = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_model.pt"), map_location=device)
model.beam_size = config_model['beam_size']
model.device = device
model.to(device)
output_dir = config_model['checkpoint']
model_to_load = model.module if hasattr(model, 'module') else model
model_to_load.load_state_dict(torch.load(output_dir, map_location=device))
model.max_length = 10
return model, tokenizer
def predict(java_snippet: str, device, model, tokenizer, config_model):
# tokenize code and other code to get source_ids
pred = model(source_ids)
return pred
@serve.deployment
class MyRunner:
def __init__(self):
with open('config.json') as f:
self.config_model = json.loads(f.read())
self.device = torch.device("cpu")
self.model = load_model_local(self.config_model, self.device)
self.model.eval()
async def __call__(self, request):
payload = await request.json()
print(f"Received request with data {payload}")
prediction = predict(payload["snippet"], self.device, self.model, self.tokenizer, self.config_model)
return {"result": prediction}
if __name__ == "__main__":
HEAD_NODE_IP_ADDRESS = os.environ.get('HEAD_NODE_IP_ADDRESS')
HEAD_NODE_IP_PORT = os.environ.get('HEAD_NODE_IP_PORT')
print(f'Connecting to head ray://{HEAD_NODE_IP_ADDRESS}:{HEAD_NODE_IP_PORT}')
ray.init(
address=f'ray://{HEAD_NODE_IP_ADDRESS}:{HEAD_NODE_IP_PORT}',
namespace="my-server",
ignore_reinit_error=True,
#runtime_env=runtime_env
) # Connect to the local running Ray cluster.
serve.run(MyRunner.bind(), host='0.0.0.0', port=80)
I want to send requests smth like this (80 or 443 port):
curl --cacert my_server.crt -i -X POST http://<some external IP>/ -H "Content-Type: application/json; indent=4" --data-binary "@test.json"