How I can use SSL for deployment? Here is my code:
import requests
import torch
import json
from transformers import RobertaModel, RobertaConfig, RobertaTokenizer
import os
import numpy as np
from torch.utils.data import DataLoader, Dataset, SequentialSampler, TensorDataset
from ray import serve
import ray
from fastapi import FastAPI
from pathlib import Path
# 1: Define a FastAPI app and wrap it in a deployment with a route handler.
app = FastAPI()
def load_model_local(config_model, device):
# build model
tokenizer = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_tokenizer.pt"), map_location=device)
config = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_config.pt"))
config.is_decoder = True
model = torch.load(os.path.join(config_model['pretrained_model_dir'], "my_model.pt"), map_location=device)
model.beam_size = config_model['beam_size']
model.device = device
model.to(device)
output_dir = config_model['checkpoint']
model_to_load = model.module if hasattr(model, 'module') else model
model_to_load.load_state_dict(torch.load(output_dir, map_location=device))
model.max_length = 10
return model, tokenizer
def predict(java_snippet: str, device, model, tokenizer, config_model):
# tokenize code and other code to get source_ids
pred = model(source_ids)
return pred
@serve.deployment
class MyRunner:
def __init__(self):
with open('config.json') as f:
self.config_model = json.loads(f.read())
self.device = torch.device("cpu")
self.model = load_model_local(self.config_model, self.device)
self.model.eval()
async def __call__(self, request):
payload = await request.json()
print(f"Received request with data {payload}")
prediction = predict(payload["snippet"], self.device, self.model, self.tokenizer, self.config_model)
return {"result": prediction}
if __name__ == "__main__":
HEAD_NODE_IP_ADDRESS = os.environ.get('HEAD_NODE_IP_ADDRESS')
HEAD_NODE_IP_PORT = os.environ.get('HEAD_NODE_IP_PORT')
print(f'Connecting to head ray://{HEAD_NODE_IP_ADDRESS}:{HEAD_NODE_IP_PORT}')
ray.init(
address=f'ray://{HEAD_NODE_IP_ADDRESS}:{HEAD_NODE_IP_PORT}',
namespace="my-server",
ignore_reinit_error=True,
#runtime_env=runtime_env
) # Connect to the local running Ray cluster.
serve.run(MyRunner.bind(), host='0.0.0.0', port=80)
How can I set SLL certificates and use TLS here?
I want to send a request like the following one:
curl --cacert server.crt -i -X POST http://7.7.7.184/ -H "Content-Type: application/json; indent=4" --data-binary "@test.json"