Getting Error Exception calling application: [Errno 5] Input/output error sometimes

my sample.proto file

syntax = "proto3";
package protofile;
import "google/protobuf/timestamp.proto";
import "google/protobuf/struct.proto";
option go_package = "./contracts/grpc";

message EventCallbackRequest {
  string event_sid = 1;
  string event_type = 2;
  string event_name = 3;
  google.protobuf.Timestamp event_timestamp = 4;
  string action_sid = 5;
  string action_custom_parameter = 6;
  google.protobuf.Struct event_data = 7;
  google.protobuf.Struct data = 8;
}

message EventCallbackResponse {
  string message = 1;
  bool success = 2;
}

service EventCallbackTriggerService {
  rpc TriggerCallbackEvent(EventCallbackRequest) returns (EventCallbackResponse) {}
}

sample_server.py

import grpc
from concurrent import futures
import sample_pb2
import sample_pb2_grpc

from google.protobuf.json_format import MessageToDict

class SampleEventHandler(sample_pb2_grpc.EventCallbackTriggerServiceServicer):
    def TriggerCallbackEvent(self, request, context):
        print("hiiiiiiii----------------")
        data = MessageToDict(request)
        print(f"data is {data}")

        return sample_pb2.EventCallbackResponse(message="Event received", success=True)

def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
    sample_pb2_grpc.add_EventCallbackTriggerServiceServicer_to_server(SampleEventHandler(), server)

    server.add_insecure_port('[::]:50051')
    print("gRPC server running on port 50051...")
    server.start()
    server.wait_for_termination()

if __name__ == '__main__':
    serve()

sample_client.py

import grpc
from google.protobuf import struct_pb2, timestamp_pb2
import sample_pb2_grpc as pb2_grpc
import sample_pb2 as pb2
from datetime import datetime


def run():
    target2 = 'grpc-test.squadstack.com:443/protofile.EventCallbackTriggerService/TriggerCallbackEvent'

    credentials = grpc.ssl_channel_credentials()

    # Establish a secure channel with Nginx (SSL and HTTP/2 for gRPC)
    with grpc.secure_channel(target2, credentials) as channel:
        # Create a stub (client) to make requests to the gRPC service
        stub = pb2_grpc.EventCallbackTriggerServiceStub(channel)

        # Prepare timestamp
        event_timestamp = timestamp_pb2.Timestamp()
        event_timestamp.FromDatetime(datetime.now())

        # Prepare sample Struct data
        event_data = struct_pb2.Struct()
        event_data["key1"] = "value1"
        event_data["key2"] = "value2"

        # Create a request with necessary event details
        request = pb2.EventCallbackRequest(
            event_sid="2K8DQQuKWiXhXdfpMnXJYKV0TdB",
            event_type="leg_lifecycle_event",
            event_name="leg_incoming",
            event_timestamp=event_timestamp,
            action_sid="action-123",
            action_custom_parameter="custom-param",
            event_data=event_data,
            data={
                "account_sid": "abcd60",
                "caller_id": "XXXX",
                "contact_uri": "YYYY",
                "date_created": "2023-01-10T10:41:39.607712401Z",
                "date_updated": "2023-01-10T10:41:39.607712401Z",
                "dc_code": "101",
                "direction": "inbound",
                "exophone": "YYYY",
                "leg_sid": "2K8DQQr2eoPW6HXJej7VesxsTUy00000",
                "network_type": "pstn",
                "nso_code": "9988fd96-ce41-11ed-9a09-0242ac110003",
                "start_time": "2023-01-10T10:41:39.039873482Z",
                "state": "ringing",
                "time_limit": 14400
            }
        )

        # Send request to the gRPC server via Nginx and receive response
        try:
            response = stub.TriggerCallbackEvent(request)
            print(f"Server response: {response.message}, Success: {response.success}")
        except grpc.RpcError as e:
            print(f"gRPC call failed: {e}")


if __name__ == '__main__':
    run()

I have setup nginx config to listen on port 50051. nginx is returning 200.

Sample response of nginx

{"level": "info","ts": "2024-09-18T08:18:14+00:00","message": "handled request POST /protofile.EventCallbackTriggerService/TriggerCallbackEvent","request": {"id": "50bb0cfe3ba03c733693c656ad331579","remote_ip": "192.168.1.95","remote_port": "18562","protocol": "HTTP/2.0","method": "POST","scheme": "https","host": "grpc-test.squadstack.com","uri": "/protofile.EventCallbackTriggerService/TriggerCallbackEvent","headers": {"user-agent": "grpc-python/1.66.1 grpc-c/43.0.0 (osx; chttp2)","accept": "","accept-encoding": "","referer": "","x-forwarded-for": "49.249.150.126","x-forwarded-proto": "https"}},"connection": {"bytes_read": 907,"bytes_sent": 124},"timing": {"duration_msecs": 0.001,"upstream_connect_time": "0.000","upstream_header_time": "0.000","upstream_response_time": "0.000"},"response": {"status": 200,"headers": {"content_length": "0","content_type": "application/grpc"}},"upstream": {"address": "127.0.0.1:50051","status": "200","response_length": "0","cache_status": ""},"ssl": {"protocol": "TLSv1.3","cipher": "TLS_AES_128_GCM_SHA256","session_reused": ".","server_name": "","client_verify": "NONE","client_s_dn": ""},"gzip_ratio": ""}

Whenever I run sample_client.py 8 out of 10 I get the following error

gRPC call failed: <_InactiveRpcError of RPC that terminated with:
    status = StatusCode.UNKNOWN
    details = "Exception calling application: [Errno 5] Input/output error"
    debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"Exception calling application: [Errno 5] Input/output error", grpc_status:2, created_time:"2024-09-18T14:41:17.38932+05:30"}"
>
On 

success, I receive

Server response: Event received, Success: True

I tried searching for Input/Output error but I did not find anything useful that could help me solving the problem. There is enough disk space in my server.

We are using ubuntu server to run sample_server.py. I am running sample_client on my local to the server hosted on 'grpc-test.squadstack.com:443/protofile.EventCallbackTriggerService/TriggerCallbackEvent'.

Requirements

grpcio==1.66.1
grpcio-tools==1.66.1

Python - 3.9 in both client and server

Can someone please help.

I checked nginx configuration, it is returning 200. Getting this Input/Output error is completely random and I am not able to understand the reason behind this.

Ideally this should return success every time if server port is up and running on 50051 port.