Background: I am trying to implement multicore processing in a python 3.4 PyQT5 application.
In the application I have numpy.ndarrays of frames, imagine it like a [n,m,t] array. I need to process each [n,m,:] array and using multicore would linearly speed up my process.
I've played with multiprocessing and got a simple script together using part of example scripts and gave me the following idea:
Simple No GUI code:
import multiprocessing as mp
import numpy
aa = numpy.random.rand(4,2,3)
def random_function(x):
return x,x**3
if __name__ == '__main__':
pool = mp.Pool(processes=4)
#with apply_asynch
#results = [pool.apply_async(cube, args=(aa[:,:,x],)) for x in range(0,aa.shape[2])]
#output = [p.get() for p in results]
#test_va = numpy.asarray( output)
#with apply
results = [pool.apply(random_function, args=(aa[:,:,x],)) for x in range(0,aa.shape[2])]
test_va = numpy.asarray( results)
This works and does about what I need it to do.
Issue: Now when I implement this in PyQT5 I incur in the "pickling" problem . So following suggestions for PyQT4 here I've made a simple GUI, spawn a thread and use multiprocessing. As a result I've got the same GUI replicated 4 times and it just does not seem to work.
PyQT5 GUI non working code:
import sys, time
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
import multiprocessing as mp
import numpy
class WorkThread(QThread):
finished = pyqtSignal(int,object)
def __del__(self):
self.wait()
def cube(x):
return x,x**3
def run(self):
aa = numpy.random.rand(4,2,3)
pool = mp.Pool(processes=4)
results = [pool.apply_async(self.cube, args=(aa[:,:,x],)) for x in range(0,aa.shape[2])]
output = [p.get() for p in results]
test_va = numpy.asarray( output)
for i in range(5):
QThread.sleep(0.3) # artificial time delay
self.finished.emit(i,test_va)
class test_multicore(QWidget):
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.setGeometry(300, 300, 280, 600)
self.setWindowTitle('Qthreads and multicore')
self.layout = QVBoxLayout(self)
self.testButton = QPushButton("test")
self.testButton.clicked.connect(self.test)
self.listwidget = QListWidget(self)
self.layout.addWidget(self.testButton)
self.layout.addWidget(self.listwidget)
self.threadPool = []
def add(self, text,random_matrix):
""" Add item to list widget """
print ("Add: " + str(text) +str(random_matrix))
self.listwidget.addItem(str(text))
self.listwidget.sortItems()
def addBatch(self,text="text",iters=6,delay=0.3):
""" Add several items to list widget """
for i in range(iters):
time.sleep(delay) # artificial time delay
self.add(text+" "+str(i), 0)
def test(self):
self.listwidget.clear()
self.addBatch("_non_thread_entries",iters=6,delay=0.3)
self.workThread = WorkThread()
self.workThread.finished[int,object].connect(self.add)
self.workThread.start()
# run
app = QApplication(sys.argv)
test = test_multicore()
test.show()
app.exec_()
I've also tried using Qobject and passing it to a thread with moveToThread but got the same issue again.
Question:
How do I implement multicore processing in my Python 3.4 PyQT5 application? Consider that I will deploy using cx_freeze on Windows and Mac.