Lerking/h2inc-old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Working on multiprocessing

Browse Source
This commit is contained in:
Jan Lerking
2018-05-09 21:05:29 +02:00
parent e0683c4670
commit 147bdc8a65
8 changed files with 246 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
Gui test/__pycache__/globvar.cpython-35.pyc
View File

Binary file not shown.

BIN
Gui test/__pycache__/h2inc_mp.cpython-35.pyc
View File

Binary file not shown.

8
Gui test/globvar.py
View File

@@ -6,9 +6,17 @@
# for direct usage in assembly programming using nasm/yasm.
from pathlib import Path
import multiprocessing
def init():
global home
home = str(Path.home())
global num_cores
num_cores = multiprocessing.cpu_count()
global queue
queue = 0
global fraction
fraction = 0
global worker
worker = []
global listener

1
Gui test/h2inc.glade
View File

@@ -38,6 +38,7 @@
<object class="GtkApplicationWindow" id="window">
<property name="can_focus">False</property>
<property name="title" translatable="yes">h2inc</property>
<property name="resizable">False</property>
<child>
<object class="GtkBox" id="box1">
<property name="visible">True</property>

3
Gui test/h2inc.glade~
View File

@@ -37,8 +37,7 @@
</object>
<object class="GtkApplicationWindow" id="window">
<property name="can_focus">False</property>
<property name="title" translatable="yes">H2INC</property>
<property name="hide_titlebar_when_maximized">True</property>
<property name="title" translatable="yes">h2inc</property>
<child>
<object class="GtkBox" id="box1">
<property name="visible">True</property>

11
Gui test/h2inc_gtk.py
View File

@@ -11,7 +11,6 @@ import sys
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gio, GObject as gobject
#import time
import globvar
from h2inc_mp import start_workers
from h2inc_fp import sourcedir_filecnt, sourcedir_foldercnt
@@ -42,7 +41,7 @@ class H2INC:
self.obj = builder.get_object
self.obj("window").set_application(app)
self.obj("window").set_wmclass("h2inc","h2inc")
self.obj("window").set_title("h2inc - v.0.0.1")
self.obj("window").set_title("h2inc - v.0.1.1")
self.obj("window").show_all()
self.obj("source_entry").set_text(globvar.defsrc)
self.obj("destination_entry").set_text(globvar.defdest)
@@ -61,8 +60,9 @@ class H2INC:
self.app.run(argv)
def callbackDisplay(self, obj, fraction, text, index, data=None):
self.obj("progress_label").set_text("{} of {}".format(index+1, self.filecnt))
self.obj("progressbar").set_fraction(fraction)
self.new_value = self.obj("progressbar").get_fraction()+fraction
self.obj("progress_label").set_text("{} of {}".format(index+1, globvar.filecnt))
self.obj("progressbar").set_fraction(self.new_value)
def callbackFinished(self, obj, data=None):
if self.process==None:
@@ -120,6 +120,7 @@ class H2INC:
elif response == Gtk.ResponseType.CANCEL:
print("Cancel")
globvar.fraction = 1/globvar.filecnt
dialog.destroy()
def on_destination_button_clicked(self,widget):
@@ -180,6 +181,8 @@ class H2INC:
print(globvar.foldercnt)
self.obj("numfolders_label").set_text(str(globvar.foldercnt))
globvar.fraction = 1/globvar.filecnt
def on_translate_button_clicked(self, widget, data=None):
self.obj("sourceframe").set_sensitive(False)
self.obj("translate_button").set_sensitive(False)

268
Gui test/h2inc_gtk_old.py Normal file → Executable file
View File

@@ -7,25 +7,167 @@
# Program to convert C-header (*.h) files to nasm include files (*.inc),
# for direct usage in assembly programming using nasm/yasm.
import os
import sys
import io
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gio, GObject as gobject
#import time
import globvar
from h2inc_mp import start_workers
from h2inc_fp import sourcedir_filecnt, sourcedir_foldercnt
import time
from multiprocessing import Process, Value, Lock, Pool, Queue
import threading
globvar.init()
gobject.threads_init()
class H2INC:
class Listener(gobject.GObject):
__gsignals__ = {
'updated' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
(gobject.TYPE_FLOAT, gobject.TYPE_STRING, gobject.TYPE_INT)),
'finished': (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
())
}
def __init__(self, queue):
gobject.GObject.__init__(self)
self.queue = queue
def go(self):
print("Listener has started")
while True:
# Listen for results on the queue and process them accordingly
data = self.queue.get()
# Check if finished
if data[1]=="finished":
print("Listener is finishing.")
self.emit("finished")
return
else:
self.emit('updated', data[0], data[1], data[2])
gobject.type_register(Listener)
class Worker():
def __init__(self, queue, filecnt, filelist):
self.queue = queue
self.filecnt = filecnt
def go(self):
print("The worker has started doing some work (counting from 0 to 9)")
for i in range(self.filecnt):
proportion = (float(i+1))/self.filecnt
self.queue.put((proportion, "working...", i))
time.sleep(0.01)
process_file(filelist[i])
self.queue.put((1.0, "finished"))
print("The worker has finished.")
tupline = []
preproc = ()
filelist = []
folderlist = []
cnt = 0
srcdir = ''
destdir = ''
#num_cores = multiprocessing.cpu_count()
fileindex = 0
filecnt = 0
incinc = ''
defdir = False
defsrc = 'usr/include'
defdest = '~'
count = 0
def sourcedir_filecnt(sourcedir):
### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
cnt = 0
global filelist
global srcdir
srcdir = sourcedir
for folderName, subfolders, files in os.walk(sourcedir):
for file in files:
if file.lower().endswith('.h'):
cnt += 1
filelist += [folderName + '/' + file]
print(folderName + '/' + file)
# print(filelist)
return cnt
def sourcedir_foldercnt(sourcedir):
### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
global cnt
global folderlist
for folderName, subfolders, files in os.walk(sourcedir):
if subfolders:
for subfolder in subfolders:
sourcedir_foldercnt(subfolder)
tempf = [file for file in files if file.lower().endswith('.h')]
if tempf:
cnt = cnt + 1
# print(folderName)
folderlist += [folderName]
# print(folderlist)
# print(len(folderlist))
return cnt
def process_file(data):
global count
outfile = ''
inputfile = data
encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865',
'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006',
'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254',
'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004',
'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp',
'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3',
'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4',
'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab',
'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek',
'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154',
'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be',
'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
for e in encodings:
try:
fh = io.open(data, 'r', encoding=e)
fh.readlines()
fh.seek(0)
except UnicodeDecodeError:
print('got unicode error with %s , trying different encoding' % e)
else:
# print('opening the file with encoding: %s ' % e)
break
# print(os.path.basename(data))
for lines in fh:
outfile = outfile + lines
fh.close()
outputfile = os.path.splitext(inputfile)[0] + '.inc'
outputfile = str(outputfile).replace(srcdir, destdir)
count += 1
print(str(count)+'->'+outputfile)
if not os.path.exists(os.path.dirname(outputfile)):
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as exc: # Guard against race condition
if exc.errno != FileExistsError:
raise
newfile = open(outputfile, "w")
newfile.write(outfile)
newfile.close()
class ExampleApp:
global app
global destlabel
def __init__(self):
self.filecnt = 0
self.foldercnt = 0
self.fileindex = 0
self.process = None
self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0))
@@ -41,13 +183,12 @@ class H2INC:
self.obj = builder.get_object
self.obj("window").set_application(app)
self.obj("window").set_wmclass("h2inc","h2inc")
self.obj("window").set_title("h2inc - v.0.0.1")
self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk")
self.obj("window").show_all()
self.obj("source_entry").set_text(globvar.defsrc)
self.obj("destination_entry").set_text(globvar.defdest)
self.obj("default_dir_checkbutton").set_active(False)
self.obj("default_dir_checkbutton").emit("toggled")
self.obj("include_checkbutton").set_active(True)
self.obj("include_checkbutton").emit("toggled")
button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL)
button.set_property("can-default",True)
@@ -95,34 +236,38 @@ class H2INC:
self.on_dialog_close(widget)
def on_source_button_clicked(self,widget):
dialog = Gtk.FileChooserDialog("Select source directory!",
self.obj("window"),
Gtk.FileChooserAction.SELECT_FOLDER,
(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
Gtk.STOCK_APPLY, Gtk.ResponseType.OK))
dialog.set_default_size(600, 300)
Gtk.FileChooser.set_filename(dialog, globvar.srcdir)
Gtk.FileChooser.set_filename(dialog, srcdir)
response = dialog.run()
if response == Gtk.ResponseType.OK:
globvar.filecnt = sourcedir_filecnt(dialog.get_filename())
if globvar.filecnt >0:
print(globvar.filecnt)
self.filecnt = sourcedir_filecnt(dialog.get_filename())
if self.filecnt >0:
print(self.filecnt)
self.obj("source_entry").set_text(dialog.get_filename())
self.obj("destination_label").set_sensitive(True)
self.obj("destination_entry").set_sensitive(True)
self.obj("destination_button").set_sensitive(True)
self.obj("numfiles_label").set_text(str(self.filecnt))
self.obj("progress_label").set_text("{} of {}".format(globvar.fileindex, globvar.filecnt))
globvar.foldercnt = sourcedir_foldercnt(dialog.get_filename())
if globvar.foldercnt >0:
self.obj("numfolders_label").set_text(str(globvar.foldercnt))
self.obj("progress_label").set_text("{} of {}".format(fileindex, self.filecnt))
foldercnt = sourcedir_foldercnt(dialog.get_filename())
if foldercnt >0:
self.obj("numfolders_label").set_text(str(foldercnt))
elif response == Gtk.ResponseType.CANCEL:
print("Cancel")
dialog.destroy()
def on_destination_button_clicked(self,widget):
global destdir
global incinc
dialog = Gtk.FileChooserDialog("Select destination directory!",
self.obj("window"),
Gtk.FileChooserAction.SELECT_FOLDER,
@@ -132,59 +277,82 @@ class H2INC:
response = dialog.run()
if response == Gtk.ResponseType.OK:
globvar.destdir = dialog.get_filename()
destdir = dialog.get_filename()
self.obj("destination_entry").set_text(dialog.get_filename())
self.obj("include_checkbutton").set_sensitive(True)
self.obj("translation_frame").set_sensitive(True)
print(globvar.srcdir)
print(srcdir)
if self.obj("include_checkbutton").get_active() == True:
globvar.incinc = '/include'
globvar.destdir = globvar.destdir+globvar.incinc
print(globvar.destdir)
incinc = '/include'
destdir = destdir+incinc
print(destdir)
elif response == Gtk.ResponseType.CANCEL:
print("Cancel")
dialog.destroy()
def on_include_checkbutton_toggled(self, widget):
global destdir
global incinc
if self.obj("include_checkbutton").get_active() == True:
if globvar.destdir.lower().endswith('/include'):
globvar.destdir = globvar.destdir
else:
globvar.destdir = globvar.destdir+'/include'
incinc = '/include'
destdir = destdir+incinc
if self.obj("include_checkbutton").get_active() == False:
#globvar.incinc = ''
globvar.destdir = globvar.destdir.replace('/include', '')
self.obj("destination_entry").set_text(globvar.destdir)
print(globvar.destdir)
incinc = ''
destdir = destdir.replace('/include', '')
self.obj("destination_entry").set_text(destdir)
print(destdir)
def on_default_dir_checkbutton_toggled(self, widget):
globvar.defdir = self.obj("default_dir_checkbutton").get_active()
if globvar.defdir == True:
globvar.srcdir = globvar.defsrc
self.obj("source_entry").set_text(globvar.srcdir)
globvar.destdir = globvar.defdest
self.obj("destination_entry").set_text(globvar.destdir)
global defdir
global srcdir
global destdir
global incinc
defdir = self.obj("default_dir_checkbutton").get_active()
if defdir == True:
srcdir = defsrc
self.obj("source_entry").set_text(srcdir)
destdir = defdest+incinc
self.obj("destination_entry").set_text(destdir)
self.obj("include_checkbutton").set_sensitive(True)
self.obj("source_label").set_sensitive(False)
self.obj("source_entry").set_sensitive(False)
self.obj("source_button").set_sensitive(False)
self.obj("translation_frame").set_sensitive(True)
globvar.filecnt = sourcedir_filecnt(globvar.srcdir)
if globvar.filecnt >0:
print(globvar.filecnt)
self.obj("numfiles_label").set_text(str(globvar.filecnt))
self.obj("progress_label").set_text("{} of {}".format(globvar.fileindex, self.filecnt))
globvar.foldercnt = sourcedir_foldercnt(globvar.srcdir)
if globvar.foldercnt >0:
print(globvar.foldercnt)
self.obj("numfolders_label").set_text(str(globvar.foldercnt))
filecnt = sourcedir_filecnt(defsrc)
if filecnt >0:
print(filecnt)
foldercnt = sourcedir_foldercnt(defsrc)
if foldercnt >0:
self.obj("numfolders_label").set_text(str(foldercnt))
def on_translate_button_clicked(self, widget, data=None):
self.obj("sourceframe").set_sensitive(False)
self.obj("translate_button").set_sensitive(False)
start_workers()
if self.process!=None:
return
app = H2INC()
app.run(sys.argv)
print("Creating shared Queue")
queue = Queue()
print("Creating Worker")
worker = Worker(queue, self.filecnt, filelist)
print("Creating Listener")
listener = Listener(queue)
listener.connect("updated",self.callbackDisplay)
listener.connect("finished",self.callbackFinished)
print("Starting Listener")
thread = threading.Thread(target=listener.go, args=())
thread.start()
print("Starting Worker")
self.process = Process(target=worker.go, args=())
self.process.start()
app = ExampleApp()
app.run(sys.argv)

21
Gui test/h2inc_mp.py
View File

@@ -11,10 +11,9 @@ from threading import Thread
import threading
import globvar
import h2inc_fp
import time
num_cores = multiprocessing.cpu_count()
lock = threading.Lock()
queue = 0
def process_queue():
print("Worker {} working...".format(threading.current_thread().name))
@@ -26,19 +25,19 @@ def process_queue():
#self.queue.put((1.0, "finished"))
#print("The worker has finished.")
while True:
cfile = queue.get()
cfile = globvar.queue.get()
print(cfile)
h2inc_fp.process_file(cfile)
queue.task_done()
globvar.queue.task_done()
def start_workers():
global queue
print("Creating shared Queue")
queue = Queue()
ts = time.time()
globvar.queue = Queue()
print("Number of cores:", num_cores)
print("Number of cores:", globvar.num_cores)
for n in range(num_cores):
for n in range(globvar.num_cores):
print("Creating Worker", n)
worker = threading.Thread(target=process_queue)
@@ -47,8 +46,10 @@ def start_workers():
for cfile in globvar.filelist:
print("Queueing {}".format(cfile))
queue.put(cfile)
globvar.queue.put(cfile)
queue.join()
globvar.queue.join()
te = time.time()-ts
print("Processing time: ", te)
return