How to apply compression while uploading file using files argument with request.post method

Question

With Flask server configured to support compression:

server.py

import os 
import json 
import tempfile
from flask import Flask, request
from flask_compress import Compress

app = Flask(__name__)
app.config['COMPRESS_MIMETYPES'] = set(['text/html', 'text/css', 'text/xml', 'application/json', 'application/javascript'])  
app.config['COMPRESS_LEVEL'] = 6
app.config['COMPRESS_MIN_SIZE'] = 500
Compress(app)


@app.route('/', methods=['GET', 'POST'])
def index():
    print('get_json: %s get_data: %s' % (type(request.get_json()), type(request.get_data())) )

    for key, value in request.files.items():
        if value.content_type == 'application/json':
            data = json.loads(value.stream.read())
            print('got json data %s' % data)

        elif value.content_type == 'application/octet-stream':
            dst_filepath = os.path.join(tempfile.mktemp(), value.filename)
            if not os.path.exists(os.path.dirname(dst_filepath)):
                os.makedirs(os.path.dirname(dst_filepath))

            with open(dst_filepath, 'wb') as fd:
                for chunk in value: 
                    fd.write(chunk)
                print('saved file as %s' % dst_filepath)

    return 'OK', 200

app.run('0.0.0.0', 80)

the client.py sends a singe request with a json dictionary and a file data.

client.py

import requests 
import json
import os 

payload = {"param_1": "value_1", "param_2": "value_2"}

filepath = '/file/path/to/local_file.zip'

data = {'json': ('some-json', json.dumps(payload), 'application/json'),
        'file': (os.path.basename(filepath), open(filepath, 'rb'), 'application/octet-stream')}

response = requests.post('http://127.0.0.1:80/', files = data)

To minimize the upload time I would like to send the request with the file data (along with the json data) compressed. How to achieve it?

Edited later:

It looks like there is a headers argument for request.post() method that could be used to specify the encoding type.

response = requests.post('http://127.0.0.1:80/', 
                          files = data,
                          headers = {'Accept-Encoding': 'gzip'})

But how do we compress the data dictionary that is used to store the file data and the json dictionary?

alphanumeric · Accepted Answer · 2019-08-18 14:34:16Z

The solution uses gzip module to compress the file's binary data when the file is sent by the client and to uncompress when the sent compressed file data is received by the server.

server.py

#!/usr/bin/env python
import os 
import io 
import json 
import gzip
import hashlib
import tempfile
from flask import Flask, request

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
    print('get_json: %s get_data: %s' % (type(request.get_json()), type(request.get_data())) )

    for key, file_storage in request.files.items():

        if file_storage.content_type == 'application/json':
            data = json.loads(file_storage.stream.read())
            print('got json data %s' % data)

        elif file_storage.content_type == 'application/octet-stream':

            zip_str = ''.join([chunk for chunk in file_storage])
            zip_str_md5_hash = hashlib.md5(zip_str).hexdigest()

            zip_filepath = os.path.join(tempfile.mktemp(), file_storage.filename)

            if not os.path.exists(os.path.dirname(zip_filepath)):
                os.makedirs(os.path.dirname(zip_filepath))

            with open(zip_filepath, 'wb') as fd:
                for chunk in zip_str: 
                    fd.write(chunk)

            filepath = zip_filepath.replace('.zip', '')

            zip_str_file = gzip.GzipFile(fileobj=io.BytesIO(zip_str)) 
            with open(filepath, 'wb') as fd:
                for chunk in zip_str_file.read():
                    fd.write(chunk)

            print('compressed str as zip_str: %s' % hashlib.md5(zip_str).hexdigest())
            print('read compressed data as zip_filepath: %s' % hashlib.md5(open(zip_filepath, 'rb').read()).hexdigest()) 
            print('read uncompressed filepath: %s' % hashlib.md5(open(filepath, 'rb').read()).hexdigest())
            print('uncompressed zip_str as zip_str_file: %s' % hashlib.md5(zip_str_file.read()).hexdigest())

    return 'OK', 200

app.run('0.0.0.0', 80)

client.py

import requests 
import json
import os 
import gzip
import io
import hashlib

payload = {"param_1": "value_1", "param_2": "value_2"}

filepath = '/file/path/to/local_file.zip'

bytes_io = io.BytesIO()
gzip_file = gzip.GzipFile(fileobj=bytes_io, mode='w')
open_file = open(filepath, 'rb')
gzip_file.write(bytes(open_file.read()))
gzip_file.close()

zip_str = bytes_io.getvalue()

data = {'json': ('some-json', json.dumps(payload), 'application/json'),
        'file': ('%s.zip'%os.path.basename(filepath), zip_str, 'application/octet-stream')}

response = requests.post('http://127.0.0.1:80', files = data)

print('file open read data as filepath %s' % hashlib.md5(open(filepath, 'rb').read()).hexdigest())
print('bytes_io%s' % hashlib.md5(bytes_io.getvalue()).hexdigest())
print('compressed str as zip_str %s' % hashlib.md5(zip_str).hexdigest())

Collectives™ on Stack Overflow

How to apply compression while uploading file using files argument with request.post method

server.py

client.py

Edited later:

1 Answer 1

server.py

client.py

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

server.py

client.py

Edited later:

1 Answer 1

server.py

client.py

Comments

Your Answer

Sign up or log in

Post as a guest

Related